okami.mom/web-extract-go
git clone https://git.nkp.pet/okami.mom/web-extract-go
70a1e29
main
1 2"""SystemOne-compatible decision server for the fine-tuned EmbeddingGemma 2 typed-decision model. 3 4Accepts web-extract's own request shapes and renders them exactly as build_data.py did for 5training: one state per passage or search result ({query, passage|search result}), the 6question text with its JSON field names replaced by words. Listens on loopback only.""" 7import json ,os ,sys ,threading ,time 8from http .server import BaseHTTPRequestHandler ,ThreadingHTTPServer 9import torch 10from transformers import AutoModel 11 12CKPT = os .environ .get ("CKPT_DIR" ,"ckpt-a100" ) 13MODEL_NAME = os .environ .get ("MODEL_NAME" ,"egemma-td-a100" ) 14LISTEN = os .environ .get ("LISTEN" ,"127.0.0.1:8796" ) 15DEV = os .environ .get ("DEVICE" )or ("mps" if torch .backends .mps .is_available ()else "cpu" ) 16WF = "web_search_evidence" 17torch .set_num_threads (int (os .environ .get ("THREADS" ,"4" ))) 18model = AutoModel .from_pretrained (CKPT ,trust_remote_code = True ,dtype = torch .float32 ).to (DEV ).eval () 19model .config .max_seq_length = 256 20lock = threading .Lock () 21 22def strip (q ): 23return q .replace ("`passage.text`" ,"the passage" ).replace ("`query`" ,"the query" ).replace ("`candidate`" ,"the search result" ).replace ("`evidence`" ,"the evidence" ) 24 25def passage_state (q ,p ): 26return {"query" :q ,"passage" : {"title" :p .get ("title" ,"" ),"heading" :p .get ("heading" ,"" ),"text" :p .get ("text" ,"" )}} 27 28def plan (req ): 29"""[(state, {question_id: question})] grouped so each state is encoded once.""" 30state ,questions = req .get ("state" )or {},req .get ("questions" )or {} 31q = state .get ("query" ,"" )if isinstance (state ,dict )else "" 32groups = {} 33def add (st ,qid ,question ): 34key = json .dumps (st ,sort_keys = True ) 35groups .setdefault (key , (st , {}))[1 ][qid ]= question 36for qid ,spec in questions .items (): 37ins = spec .get ("instructions" ) 38if isinstance (ins ,dict )and "passage" in ins : 39add (passage_state (q ,ins ["passage" ]),qid , {"type" :"noul" ,"instructions" :strip (ins ["question" ])}) 40elif isinstance (ins ,dict )and "candidate" in ins : 41c = ins ["candidate" ] 42add ({"query" :q ,"search result" : {"title" :c .get ("title" ,"" ),"url" :c .get ("url" ,"" ),"snippet" :c .get ("snippet" ,"" )}}, 43qid , {"type" :"noul" ,"instructions" :strip (ins ["question" ])}) 44elif isinstance (state ,dict )and "passage" in state : 45add (passage_state (q ,state ["passage" ]),qid , {** spec ,"instructions" :strip (str (ins ))}) 46else : 47add (state ,qid , {** spec ,"instructions" :strip (str (ins ))if isinstance (ins ,str )else ins }) 48return list (groups .values ()) 49 50def answer (req ): 51"""One forward pass for every state and question in the request (decide() does one state per pass).""" 52out = {} 53groups = plan (req ) 54if not groups : 55return {"model" :MODEL_NAME ,"answers" :out ,"usage" : {"input_tokens" :0 ,"output_tokens" :0 }} 56with lock ,torch .inference_mode (): 57inputs ,specs ,_ = model ._build_inputs ([(WF ,st ,qs ,None )for st ,qs in groups ]) 58inputs = {k :v .to (model .device )if torch .is_tensor (v )else v for k ,v in inputs .items ()} 59probs = model (** inputs ).probabilities .cpu () 60for i ,spec in enumerate (specs ): 61p = dict (zip (spec .options ,probs [i , :len (spec .options )].tolist ())) 62if spec .kind == "noul" : 63out [spec .qid ]= {"type" :"noul" ,"noul" :round (p ["true" ],4 )} 64else : 65label = max (p ,key = p .get ) 66out [spec .qid ]= {"type" :spec .kind ,"choice" :label ,"confidence" :round (p [label ],4 ), 67"probabilities" : {k :round (v ,4 )for k ,v in p .items ()}} 68return {"model" :MODEL_NAME ,"answers" :out ,"usage" : {"input_tokens" :0 ,"output_tokens" :0 }} 69 70class Handler (BaseHTTPRequestHandler ): 71def _send (self ,code ,body ): 72data = json .dumps (body ).encode () 73self .send_response (code );self .send_header ("content-type" ,"application/json" );self .send_header ("content-length" ,str (len (data )));self .end_headers ();self .wfile .write (data ) 74def do_GET (self ): 75self ._send (200 , {"ok" :True ,"model" :MODEL_NAME ,"device" :DEV })if self .path == "/healthz" else self ._send (404 , {"error" :"not found" }) 76def do_POST (self ): 77if self .path .rstrip ("/" )not in ("/systemone" ,"/v1/systemone" ): 78return self ._send (404 , {"error" :"not found" }) 79try : 80n = int (self .headers .get ("content-length" ,"0" )) 81if n > 2_000_000 :return self ._send (413 , {"error" :"too large" }) 82t0 = time .time ();body = answer (json .loads (self .rfile .read (n ))) 83self ._send (200 ,body ) 84sys .stderr .write (f" { len ( body [ 'answers' ]) } answers in { time . time () - t0 :.2f } s\n" ) 85except Exception as e : 86self ._send (500 , {"error" :str (e )[:300 ]}) 87def log_message (self ,* a ):pass 88 89host ,port = LISTEN .rsplit (":" ,1 ) 90f"serving { MODEL_NAME } on { LISTEN } ( { DEV } )" ,flush = True ) 91ThreadingHTTPServer ((host ,int (port )),Handler ).serve_forever ()