"""SystemOne-compatible decision server for the fine-tuned EmbeddingGemma 2 typed-decision model. Accepts web-extract's own request shapes and renders them exactly as build_data.py did for training: one state per passage or search result ({query, passage|search result}), the question text with its JSON field names replaced by words. Listens on loopback only.""" import json, os, sys, threading, time from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer import torch from transformers import AutoModel CKPT = os.environ.get("CKPT_DIR", "ckpt-a100") MODEL_NAME = os.environ.get("MODEL_NAME", "egemma-td-a100") LISTEN = os.environ.get("LISTEN", "127.0.0.1:8796") DEV = os.environ.get("DEVICE") or ("mps" if torch.backends.mps.is_available() else "cpu") WF = "web_search_evidence" torch.set_num_threads(int(os.environ.get("THREADS", "4"))) model = AutoModel.from_pretrained(CKPT, trust_remote_code=True, dtype=torch.float32).to(DEV).eval() model.config.max_seq_length = 256 lock = threading.Lock() def strip(q): return q.replace("`passage.text`", "the passage").replace("`query`", "the query").replace("`candidate`", "the search result").replace("`evidence`", "the evidence") def passage_state(q, p): return {"query": q, "passage": {"title": p.get("title", ""), "heading": p.get("heading", ""), "text": p.get("text", "")}} def plan(req): """[(state, {question_id: question})] grouped so each state is encoded once.""" state, questions = req.get("state") or {}, req.get("questions") or {} q = state.get("query", "") if isinstance(state, dict) else "" groups = {} def add(st, qid, question): key = json.dumps(st, sort_keys=True) groups.setdefault(key, (st, {}))[1][qid] = question for qid, spec in questions.items(): ins = spec.get("instructions") if isinstance(ins, dict) and "passage" in ins: add(passage_state(q, ins["passage"]), qid, {"type": "noul", "instructions": strip(ins["question"])}) elif isinstance(ins, dict) and "candidate" in ins: c = ins["candidate"] add({"query": q, "search result": {"title": c.get("title", ""), "url": c.get("url", ""), "snippet": c.get("snippet", "")}}, qid, {"type": "noul", "instructions": strip(ins["question"])}) elif isinstance(state, dict) and "passage" in state: add(passage_state(q, state["passage"]), qid, {**spec, "instructions": strip(str(ins))}) else: add(state, qid, {**spec, "instructions": strip(str(ins)) if isinstance(ins, str) else ins}) return list(groups.values()) def answer(req): """One forward pass for every state and question in the request (decide() does one state per pass).""" out = {} groups = plan(req) if not groups: return {"model": MODEL_NAME, "answers": out, "usage": {"input_tokens": 0, "output_tokens": 0}} with lock, torch.inference_mode(): inputs, specs, _ = model._build_inputs([(WF, st, qs, None) for st, qs in groups]) inputs = {k: v.to(model.device) if torch.is_tensor(v) else v for k, v in inputs.items()} probs = model(**inputs).probabilities.cpu() for i, spec in enumerate(specs): p = dict(zip(spec.options, probs[i, : len(spec.options)].tolist())) if spec.kind == "noul": out[spec.qid] = {"type": "noul", "noul": round(p["true"], 4)} else: label = max(p, key=p.get) out[spec.qid] = {"type": spec.kind, "choice": label, "confidence": round(p[label], 4), "probabilities": {k: round(v, 4) for k, v in p.items()}} return {"model": MODEL_NAME, "answers": out, "usage": {"input_tokens": 0, "output_tokens": 0}} class Handler(BaseHTTPRequestHandler): def _send(self, code, body): data = json.dumps(body).encode() self.send_response(code); self.send_header("content-type", "application/json"); self.send_header("content-length", str(len(data))); self.end_headers(); self.wfile.write(data) def do_GET(self): self._send(200, {"ok": True, "model": MODEL_NAME, "device": DEV}) if self.path == "/healthz" else self._send(404, {"error": "not found"}) def do_POST(self): if self.path.rstrip("/") not in ("/systemone", "/v1/systemone"): return self._send(404, {"error": "not found"}) try: n = int(self.headers.get("content-length", "0")) if n > 2_000_000: return self._send(413, {"error": "too large"}) t0 = time.time(); body = answer(json.loads(self.rfile.read(n))) self._send(200, body) sys.stderr.write(f"{len(body['answers'])} answers in {time.time()-t0:.2f}s\n") except Exception as e: self._send(500, {"error": str(e)[:300]}) def log_message(self, *a): pass host, port = LISTEN.rsplit(":", 1) print(f"serving {MODEL_NAME} on {LISTEN} ({DEV})", flush=True) ThreadingHTTPServer((host, int(port)), Handler).serve_forever()