okami.mom/web-extract-go

git clone https://git.nkp.pet/okami.mom/web-extract-go

nekomimi.petdecisions: ask a local model first and keep only its confident answers70a1e29

main
4.9 KiB91 linesraw
1
2"""SystemOne-compatible decision server for the fine-tuned EmbeddingGemma 2 typed-decision model.
3
4Accepts web-extract's own request shapes and renders them exactly as build_data.py did for
5training: one state per passage or search result ({query, passage|search result}), the
6question text with its JSON field names replaced by words. Listens on loopback only."""
7import json, os, sys, threading, time
8from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
9import torch
10from transformers import AutoModel
11
12CKPT = os.environ.get("CKPT_DIR", "ckpt-a100")
13MODEL_NAME = os.environ.get("MODEL_NAME", "egemma-td-a100")
14LISTEN = os.environ.get("LISTEN", "127.0.0.1:8796")
15DEV = os.environ.get("DEVICE") or ("mps" if torch.backends.mps.is_available() else "cpu")
16WF = "web_search_evidence"
17torch.set_num_threads(int(os.environ.get("THREADS", "4")))
18model = AutoModel.from_pretrained(CKPT, trust_remote_code=True, dtype=torch.float32).to(DEV).eval()
19model.config.max_seq_length = 256
20lock = threading.Lock()
21
22def strip(q):
23    return q.replace("`passage.text`", "the passage").replace("`query`", "the query").replace("`candidate`", "the search result").replace("`evidence`", "the evidence")
24
25def passage_state(q, p):
26    return {"query": q, "passage": {"title": p.get("title", ""), "heading": p.get("heading", ""), "text": p.get("text", "")}}
27
28def plan(req):
29    """[(state, {question_id: question})] grouped so each state is encoded once."""
30    state, questions = req.get("state") or {}, req.get("questions") or {}
31    q = state.get("query", "") if isinstance(state, dict) else ""
32    groups = {}
33    def add(st, qid, question):
34        key = json.dumps(st, sort_keys=True)
35        groups.setdefault(key, (st, {}))[1][qid] = question
36    for qid, spec in questions.items():
37        ins = spec.get("instructions")
38        if isinstance(ins, dict) and "passage" in ins:
39            add(passage_state(q, ins["passage"]), qid, {"type": "noul", "instructions": strip(ins["question"])})
40        elif isinstance(ins, dict) and "candidate" in ins:
41            c = ins["candidate"]
42            add({"query": q, "search result": {"title": c.get("title", ""), "url": c.get("url", ""), "snippet": c.get("snippet", "")}},
43                qid, {"type": "noul", "instructions": strip(ins["question"])})
44        elif isinstance(state, dict) and "passage" in state:
45            add(passage_state(q, state["passage"]), qid, {**spec, "instructions": strip(str(ins))})
46        else:
47            add(state, qid, {**spec, "instructions": strip(str(ins)) if isinstance(ins, str) else ins})
48    return list(groups.values())
49
50def answer(req):
51    """One forward pass for every state and question in the request (decide() does one state per pass)."""
52    out = {}
53    groups = plan(req)
54    if not groups:
55        return {"model": MODEL_NAME, "answers": out, "usage": {"input_tokens": 0, "output_tokens": 0}}
56    with lock, torch.inference_mode():
57        inputs, specs, _ = model._build_inputs([(WF, st, qs, None) for st, qs in groups])
58        inputs = {k: v.to(model.device) if torch.is_tensor(v) else v for k, v in inputs.items()}
59        probs = model(**inputs).probabilities.cpu()
60    for i, spec in enumerate(specs):
61        p = dict(zip(spec.options, probs[i, : len(spec.options)].tolist()))
62        if spec.kind == "noul":
63            out[spec.qid] = {"type": "noul", "noul": round(p["true"], 4)}
64        else:
65            label = max(p, key=p.get)
66            out[spec.qid] = {"type": spec.kind, "choice": label, "confidence": round(p[label], 4),
67                             "probabilities": {k: round(v, 4) for k, v in p.items()}}
68    return {"model": MODEL_NAME, "answers": out, "usage": {"input_tokens": 0, "output_tokens": 0}}
69
70class Handler(BaseHTTPRequestHandler):
71    def _send(self, code, body):
72        data = json.dumps(body).encode()
73        self.send_response(code); self.send_header("content-type", "application/json"); self.send_header("content-length", str(len(data))); self.end_headers(); self.wfile.write(data)
74    def do_GET(self):
75        self._send(200, {"ok": True, "model": MODEL_NAME, "device": DEV}) if self.path == "/healthz" else self._send(404, {"error": "not found"})
76    def do_POST(self):
77        if self.path.rstrip("/") not in ("/systemone", "/v1/systemone"):
78            return self._send(404, {"error": "not found"})
79        try:
80            n = int(self.headers.get("content-length", "0"))
81            if n > 2_000_000: return self._send(413, {"error": "too large"})
82            t0 = time.time(); body = answer(json.loads(self.rfile.read(n)))
83            self._send(200, body)
84            sys.stderr.write(f"{len(body['answers'])} answers in {time.time()-t0:.2f}s\n")
85        except Exception as e:
86            self._send(500, {"error": str(e)[:300]})
87    def log_message(self, *a): pass
88
89host, port = LISTEN.rsplit(":", 1)
90print(f"serving {MODEL_NAME} on {LISTEN} ({DEV})", flush=True)
91ThreadingHTTPServer((host, int(port)), Handler).serve_forever()