🎚️ Jev, and asking for an answer instead of a sentence · Jev กับการขอคำตอบ แทนที่จะขอประโยค · Jev：要一个答案，而不是一句话

1 One question, one number · หนึ่งคำถาม หนึ่งตัวเลข · 一个问题，一个数字

📄 NEW FILE · decide1.py

"""decide1.py — a model that answers instead of talking."""
import sys

import kev

sys.stdout.reconfigure(encoding="utf-8")

TICKETS = [
    "I was charged twice for the same order. Please refund the extra one.",
    "Just wanted to say the new dashboard is lovely. No reply needed.",
    "The app crashes when I tap export. Android 14, latest build.",
    "Can you send me an invoice for last month for my accountant?",
]

# Telling it the situation, not only the question, is the single biggest
# thing you control. The same model on the same tickets does much worse
# when handed a bare question with no setting.
SITUATION = "A support desk is reading a customer message:\n{ticket}"
ASK = "Is this customer asking for a refund?"


def main():
    """Ask one yes/no question about each ticket, and print the number."""
    for ticket in TICKETS:
        # noul is the model's word for a yes/no question. What comes back
        # is not the word "yes"; it is how much of the probability sat on
        # yes. There is no sentence to parse and nothing to strip.
        sure = kev.noul(SITUATION.format(ticket=ticket), ASK)
        bar = "#" * int(sure * 40)
        print(f"{sure:5.3f} {bar:40} {ticket[:44]}")


if __name__ == "__main__":
    main()

🖥️ WHAT IT PRINTED · four tickets, four numbers

0.991 #######################################  I was charged twice. Please refund
0.008                                          Just wanted to say the dashboard is lovely
0.027 #                                        The app crashes when I tap export
0.082 ###                                      Can you send me an invoice for last month

📄 NEW FILE · kev.py

"""kev.py — run the open decision model, and read its answers.

Jev is TypeSafe's hosted model and you need a key for it. Kev is the open
copy of the same idea, and the contract is the same: choice, score and
noul questions in, a probability for every option out.

One document, called the state. Any number of typed questions. They are
packed into a single sequence with five delimiter tokens, and the model
answers all of them in one forward pass.
"""
import json
import re
from pathlib import Path

import numpy as np
import onnxruntime as ort
from tokenizers import Tokenizer

HERE = Path(__file__).parent / "kev-0.6b-onnx"
CONFIG = json.loads((HERE / "config.json").read_text(encoding="utf-8"))

# The delimiters live in config.json, under "kev". Reading them from there
# beats writing the numbers down and hoping the next version agrees.
D = CONFIG["kev"]["delimiter_ids"]
STATE, QUESTION = D["state"], D["question"]
OPTION, OPTION_END, DECIDE = D["option_start"], D["option_end"], D["decide"]
PAD = CONFIG["pad_token_id"]

_tokenizer = Tokenizer.from_file(str(HERE / "tokenizer.json"))
_session = None

# Text from a person must never be able to fake a delimiter, so <|x|>
# becomes <¦x¦> on the way in.
_FORGE = re.compile(r"<\|([A-Za-z0-9_]+)\|>")


def session():
    """Load the graph once and keep it."""
    global _session
    if _session is None:
        _session = ort.InferenceSession(str(HERE / "onnx" / "model_q4.onnx"),
                                        providers=["CPUExecutionProvider"])
    return _session


def encode(text):
    return _tokenizer.encode(_FORGE.sub(r"<¦\1¦>", text),
                             add_special_tokens=False).ids


def pack(state, questions):
    """Lay out the state and every question, and note where to read.

    Only the logits at each option's closing token mean anything. Get
    this arithmetic wrong and you get confident nonsense, not an error.
    """
    tokens = [STATE] + encode(state)
    reads = []
    for question in questions:
        instruction = encode(question["ask"])
        spans = [[OPTION] + encode(option) + [OPTION_END]
                 for option in question["options"]]

        base = len(tokens)
        cursor = 1 + len(instruction)
        ends = []
        for span in spans:
            cursor = cursor + len(span)
            ends.append(base + cursor - 1)

        tokens.append(QUESTION)
        tokens.extend(instruction)
        for span in spans:
            tokens.extend(span)
        tokens.append(DECIDE)
        reads.append(ends)
    return tokens, reads


def softmax(values):
    shifted = np.array(values, dtype=np.float64) - np.max(values)
    weights = np.exp(shifted)
    return weights / weights.sum()


def ask(state, questions):
    """One forward pass. Returns a probability per option, per question."""
    tokens, reads = pack(state, questions)
    input_ids = np.array([tokens], dtype=np.int64)
    attention_mask = np.ones_like(input_ids)
    outputs = session().run(None, {"input_ids": input_ids,
                                   "attention_mask": attention_mask})
    logits = np.asarray(outputs[0]).reshape(-1)
    return [softmax([logits[i] for i in ends]) for ends in reads]


def noul(state, ask_text):
    """A yes/no question. Returns how sure the model is of yes."""
    # The order is the model's own, set in config.json: ["no", "yes"].
    options = CONFIG["kev"]["noul_options"]
    return float(ask(state, [{"ask": ask_text, "options": options}])[0][1])

2 Three kinds of question, one pass · คำถามสามชนิด รอบเดียว · 三种问题，一次前向

📄 NEW FILE · three1.py

"""three1.py — three kinds of question, one document, one forward pass."""
import sys
import time

import kev

sys.stdout.reconfigure(encoding="utf-8")

TICKET = ("I was charged twice for the same order and nobody answers my "
          "emails. I want my money back today or I am calling the bank.")

# The three types Jev defines, and Kev copies.
#   choice - pick one option. Describe each one; the words are the spec.
#   score  - place it on an ordered scale, written worst to best.
#   noul   - a yes/no proposition. Always the options no, then yes.
QUESTIONS = [
    {"ask": "Which team should handle this ticket?",
     "options": ["billing: charges, refunds, invoices",
                 "technical support: bugs, outages, login problems",
                 "sales: pricing and new accounts",
                 "account management: renewals and contracts"]},
    {"ask": "How angry is this customer?",
     "options": ["calm", "mildly annoyed", "frustrated", "furious"]},
    {"ask": "Is the customer threatening a chargeback?",
     "options": ["no", "yes"]},
]


def main():
    """All three questions go in together and come back together."""
    # Loading the graph takes a couple of seconds, once. Do it before the
    # clock starts, or the first measurement is mostly disk.
    kev.session()

    started = time.perf_counter()

    # One call. The state is read once, and every question sees it. Asking
    # a chat model these three things means three prompts, or one prompt
    # and a paragraph to unpick afterwards.
    answers = kev.ask(TICKET, QUESTIONS)
    spent = time.perf_counter() - started

    for question, probabilities in zip(QUESTIONS, answers):
        print(question["ask"])
        order = sorted(range(len(probabilities)),
                       key=lambda i: -probabilities[i])
        for rank in order:
            bar = "#" * int(probabilities[rank] * 30)
            print(f"   {probabilities[rank]:5.3f} {bar:30} "
                  f"{question['options'][rank]}")
        print()

    print(f"three questions, one pass, {spent * 1000:.0f} ms")


if __name__ == "__main__":
    main()

🖥️ WHAT IT PRINTED · one ticket, three answers

Which team should handle this ticket?
   0.990 #############################  billing: charges, refunds, invoices
   0.008                                technical support: bugs, outages
   0.002                                sales: pricing and new accounts

How angry is this customer?
   0.639 ###################            frustrated
   0.244 #######                        furious
   0.089 ##                             mildly annoyed
   0.028                                calm

Is the customer threatening a chargeback?
   0.821 ########################       yes

three questions, one pass, 548 ms

3 The same tickets, both models · ตั๋วชุดเดียวกัน สองโมเดล · 同样的工单，两个模型

📄 NEW FILE · tickets.py

"""tickets.py — support messages, and whether each is asking for a refund.

Written before anything was measured against them, so the numbers on this
page are not the result of choosing convenient examples.
"""

TICKETS = [
    # ── plainly asking ──────────────────────────────────────────────────
    ("I was charged twice for the same order. Please refund the extra one.", True),
    ("Cancel my subscription and give me my money back.", True),
    ("Your service did not work for a single day. I want a refund.", True),
    ("Please return the payment from 3 September, I did not authorise it.", True),

    # ── asking, without the word refund ─────────────────────────────────
    ("You took 890 baht twice. Put one back please.", True),
    ("I paid for a year and used it once. Can we undo that?", True),
    ("Send the money back to the same card, thanks.", True),

    # ── about money, but not asking for it back ─────────────────────────
    ("Can you send me an invoice for last month for my accountant?", False),
    ("Do you offer a student discount before I subscribe?", False),
    ("What is your refund policy, out of interest?", False),
    ("The price went up. Is that on the website somewhere?", False),

    # ── nothing to do with money ────────────────────────────────────────
    ("The app crashes when I tap export. Android 14, latest build.", False),
    ("Just wanted to say the new dashboard is lovely. No reply needed.", False),
    ("How do I add a second user to my team?", False),
    ("Is there an outage? Nothing has loaded since this morning.", False),
]

# Which desk each one belongs on, for the routing step. Same order.
TEAMS = ["billing", "billing", "billing", "billing",
         "billing", "billing", "billing",
         "billing", "sales", "billing", "billing",
         "technical support", "help", "help", "technical support"]

📄 NEW FILE · sure1.py

"""sure1.py — the number is the point. Is it worth anything?

Both models are asked the same fifteen tickets. Accuracy is one thing.
What matters more is whether the confidence each one reports tells you
anything you can act on.
"""
import json
import sys
import time
import urllib.request

import kev
from tickets import TICKETS

OLLAMA = "http://localhost:11434/api/chat"
MODEL = "llama3.2"

SITUATION = "A support desk is reading a customer message:\n{ticket}"
ASK = "Is this customer asking for a refund?"

PROMPT = (SITUATION + "\n\nIs this customer asking for a refund? Answer with "
          'JSON only:\n{{"refund": true or false, '
          '"confidence": a number from 0 to 1}}')

sys.stdout.reconfigure(encoding="utf-8")


def ask_llm(ticket):
    """The same question to a chat model, asking it to report confidence."""
    body = json.dumps({"model": MODEL, "stream": False, "format": "json",
                       "options": {"temperature": 0},
                       "messages": [{"role": "user",
                                     "content": PROMPT.format(ticket=ticket)}]}).encode()
    request = urllib.request.Request(OLLAMA, data=body,
                                     headers={"Content-Type": "application/json"})
    start = time.perf_counter()
    with urllib.request.urlopen(request, timeout=300) as response:
        answer = json.loads(response.read())
    spent = time.perf_counter() - start
    parsed = json.loads(answer["message"]["content"])
    return bool(parsed.get("refund", False)), float(parsed.get("confidence", 0)), spent


def main():
    kev.session()

    rows = []
    for ticket, wanted in TICKETS:
        start = time.perf_counter()
        sure = kev.noul(SITUATION.format(ticket=ticket), ASK)
        kev_time = time.perf_counter() - start
        said, confidence, llm_time = ask_llm(ticket)
        rows.append((ticket, wanted, sure, kev_time, said, confidence, llm_time))

    kev_right = sum((r[2] >= 0.5) == r[1] for r in rows)
    llm_right = sum(r[4] == r[1] for r in rows)
    print(f"kev       {kev_right}/{len(rows)} right, "
          f"{sum(r[3] for r in rows) / len(rows) * 1000:6.0f} ms each")
    print(f"llama3.2  {llm_right}/{len(rows)} right, "
          f"{sum(r[6] for r in rows) / len(rows) * 1000:6.0f} ms each")

    # The real question. A number you can set a threshold on has to take
    # more than a handful of values.
    kev_values = sorted({round(r[2], 3) for r in rows})
    llm_values = sorted({r[5] for r in rows})
    print()
    print(f"kev gave {len(kev_values)} different numbers, "
          f"from {min(kev_values):.3f} to {max(kev_values):.3f}")
    print(f"llama3.2 gave {len(llm_values)}: {llm_values}")

    print()
    print(f"{'ticket':46} {'want':>5} {'kev':>7} {'llm':>6} {'conf':>6}")
    print("-" * 76)
    for ticket, wanted, sure, _, said, confidence, _ in rows:
        mark = " " if (sure >= 0.5) == wanted else "K"
        mark = mark + (" " if said == wanted else "L")
        print(f"{mark} {ticket[:44]:44} {str(wanted):>5} {sure:7.3f} "
              f"{str(said):>6} {confidence:6.2f}")


if __name__ == "__main__":
    main()

🖥️ WHAT IT PRINTED · accuracy, then the numbers themselves

kev       10/15 right,    457 ms each
llama3.2  15/15 right,   2150 ms each

kev gave 15 different numbers, from 0.008 to 0.991
llama3.2 gave 4: [0.2, 0.8, 0.9, 1.0]

4 What a real probability is for · ความน่าจะเป็นจริง ๆ มีไว้ทำอะไร · 真正的概率是拿来做什么的

📄 NEW FILE · route1.py

"""route1.py — the choice type, and what a real probability is for.

A number between 0 and 1 is only worth having if you do something
different at different values. This sends the sure ones straight through
and puts the rest in front of a person.
"""
import sys
import time

import kev
from tickets import TEAMS as WANTED
from tickets import TICKETS

SITUATION = "A support desk is reading a customer message:\n{ticket}"

TEAMS = {"ask": "Which team should handle this message?",
         "options": ["billing: charges, refunds, invoices, prices",
                     "technical support: bugs, crashes, outages, logins",
                     "sales: discounts, new accounts, upgrades",
                     "help: how do I questions about using the product"]}

# Act above this. Below it, a person looks. Chosen by reading the numbers
# below, not by taste.
SURE_ENOUGH = 0.80

sys.stdout.reconfigure(encoding="utf-8")


def route(ticket):
    """Pick a team, and say how sure that pick was."""
    answers = kev.ask(SITUATION.format(ticket=ticket), [TEAMS])[0]
    best = max(range(len(answers)), key=lambda i: answers[i])

    # The team name is everything before the colon.
    return TEAMS["options"][best].split(":")[0], float(answers[best])


def main():
    kev.session()
    started = time.perf_counter()

    picks = []
    for (ticket, _), wanted in zip(TICKETS, WANTED):
        team, sure = route(ticket)
        picks.append((sure, team, wanted, ticket))
        if sure >= SURE_ENOUGH:
            what = "-> " + team
        else:
            what = "?? " + team + ", ask a person"
        mark = " " if team == wanted else "X"
        print(f"{mark} {sure:5.3f}  {what:34} {ticket[:38]}")

    spent = time.perf_counter() - started
    print()
    print(f"{spent / len(TICKETS) * 1000:.0f} ms per ticket")
    print()

    # The threshold is the whole product decision. Higher means fewer
    # handled on their own, and fewer mistakes let through.
    print(f"{'act above':>10} {'routed':>7} {'of those, right':>17}")
    for cut in (0.60, 0.70, 0.80, 0.90, 0.95):
        acted = [p for p in picks if p[0] >= cut]
        right = sum(1 for sure, team, wanted, _ in acted if team == wanted)
        print(f"{cut:10.2f} {len(acted):7} {right:10} of {len(acted)}")


if __name__ == "__main__":
    main()

🖥️ WHAT IT PRINTED · the sweep, at the bottom

 act above  routed   of those, right
      0.60      13         11 of 13
      0.70      10          9 of 10
      0.80       7          6 of 7
      0.90       5          5 of 5
      0.95       4          4 of 4

5 It scores the topic, not the request · มันให้คะแนนหัวข้อ ไม่ใช่คำขอ · 它打的是主题的分，不是请求的分

📄 NEW FILE · where1.py

"""where1.py — the five it got wrong, and the pattern in them.

A model that gives you a dial is only useful if you know which way it
bends. These are sure1.py's failures, sorted into the two shapes they
actually take.
"""
import sys

import kev

sys.stdout.reconfigure(encoding="utf-8")

SITUATION = "A support desk is reading a customer message:\n{ticket}"
ASK = "Is this customer asking for a refund?"

# About refunds, but not asking for one. The model scores the topic.
TOPIC_NOT_INTENT = [
    "What is your refund policy, out of interest?",
    "Do you offer a student discount before I subscribe?",
    "I read your refund terms and they seem fair.",
]

# Asking for one, without sounding like it. Plain words, no keywords.
INTENT_NOT_TOPIC = [
    "Send the money back to the same card, thanks.",
    "You took 890 baht twice. Put one back please.",
    "I paid for a year and used it once. Can we undo that?",
]

# For contrast: says it plainly, and is scored plainly.
PLAIN = ["I was charged twice. Please refund the extra one."]


def show(label, tickets, wanted):
    print(label)
    for ticket in tickets:
        sure = kev.noul(SITUATION.format(ticket=ticket), ASK)
        mark = " " if (sure >= 0.5) == wanted else "X"
        print(f"  {mark} {sure:5.3f}  {ticket}")
    print()


def main():
    kev.session()
    show("asking plainly (wants a high number)", PLAIN, True)
    show("about refunds, not asking (wants a low number)",
         TOPIC_NOT_INTENT, False)
    show("asking, but never says refund (wants a high number)",
         INTENT_NOT_TOPIC, True)


if __name__ == "__main__":
    main()

🖥️ WHAT IT PRINTED · the pattern, in three groups

asking plainly (wants a high number)
    0.994  I was charged twice. Please refund the extra one.

about refunds, not asking (wants a low number)
  X 0.979  What is your refund policy, out of interest?
  X 0.599  I read your refund terms and they seem fair.

asking, but never says refund (wants a high number)
  X 0.031  Send the money back to the same card, thanks.
  X 0.195  You took 890 baht twice. Put one back please.
  X 0.055  I paid for a year and used it once. Can we undo that?

6 The request, and what it costs · หน้าตาของคำขอ และราคาของมัน · 请求长什么样，以及它多少钱

📄 NEW FILE · jev1.py

"""jev1.py — the hosted model, and the exact shape of the request.

Everything up to here ran on your own machine with the open model. Jev is
TypeSafe's hosted one, and it needs a key. This file builds the request
either way, prints it, and only sends it if a key is there.

So you can read the contract without paying for it, and the day you do
have a key nothing about the file changes.
"""
import json
import os
import sys
import urllib.error
import urllib.request

# Jev speaks the OpenAI chat shape, with one extra field. Providers differ
# only in the address and the model name.
PROVIDERS = {
    "openrouter": ("https://openrouter.ai/api/v1/chat/completions",
                   "typesafe/jev-1.13"),
    "requesty": ("https://router.requesty.ai/v1/chat/completions",
                 "typesafe/jev-latest"),
}
WHICH = os.environ.get("JEV_PROVIDER", "openrouter")
KEY = os.environ.get("JEV_API_KEY")

TICKET = ("I was charged twice for the same order and nobody answers my "
          "emails. I want my money back today or I am calling the bank.")

sys.stdout.reconfigure(encoding="utf-8")


def build(state):
    """The request body. This is the whole difference from a chat call.

    response_format carries the questions. Each one is named, typed, and
    given its options as criteria. Jev answers all of them at once, the
    same way Kev does, because Kev copied this contract.
    """
    url, model = PROVIDERS[WHICH]
    body = {
        "model": model,
        "messages": [{"role": "user", "content": state}],
        "response_format": {
            "type": "questions",
            "questions": {
                "team": {
                    "type": "choice",
                    "instructions": "Which team should handle this ticket?",
                    "criteria": {
                        "billing": "charges, refunds, invoices",
                        "technical support": "bugs, outages, login problems",
                        "sales": "pricing and new accounts",
                    },
                },
                "anger": {
                    "type": "score",
                    "instructions": "How angry is this customer?",
                    "criteria": {
                        "calm": "no sign of irritation",
                        "annoyed": "mildly put out",
                        "frustrated": "clearly unhappy",
                        "furious": "threatening to escalate",
                    },
                },
                "chargeback": {
                    "type": "noul",
                    "instructions": "Is the customer threatening a chargeback?",
                },
            },
        },
    }
    return url, body


def main():
    url, body = build(TICKET)
    print("POST", url)
    print(json.dumps(body, indent=2)[:900])
    print()

    if not KEY:
        # Nothing to apologise for. The contract above is the lesson, and
        # it is the same contract the local model answers in step 2.
        print("No JEV_API_KEY set, so nothing was sent.")
        print("Set one from openrouter.ai or requesty.ai and run it again.")
        print("Pricing at the time of writing: $0.042 per million input")
        print("tokens, output free. A ticket like this is a few hundred.")
        return

    request = urllib.request.Request(
        url, data=json.dumps(body).encode(),
        headers={"Authorization": "Bearer " + KEY,
                 "Content-Type": "application/json"})
    try:
        with urllib.request.urlopen(request, timeout=60) as response:
            answer = json.load(response)
    except urllib.error.HTTPError as refused:
        print("refused:", refused.code, refused.read().decode()[:300])
        return

    # The answers arrive as JSON inside the assistant message, keyed by
    # the names you gave the questions.
    print(json.dumps(json.loads(answer["choices"][0]["message"]["content"]),
                     indent=2)[:700])


if __name__ == "__main__":
    main()

🖥️ WHAT IT PRINTED · with no key set

POST https://openrouter.ai/api/v1/chat/completions
{
  "model": "typesafe/jev-1.13",
  "messages": [{"role": "user", "content": "I was charged twice..."}],
  "response_format": {
    "type": "questions",
    "questions": {
      "team":       {"type": "choice", "criteria": {...}},
      "anger":      {"type": "score",  "criteria": {...}},
      "chargeback": {"type": "noul"}
    }
  }
}

No JEV_API_KEY set, so nothing was sent.

7 Try first. The answers are folded away. · ลองเองก่อน เฉลยพับซ่อนไว้ · 先自己试。答案折起来放着。

"""fix1.py — challenge 1: the question is the only dial you have.

where1.py showed the model scoring the word refund rather than the
request. There is no training here and no prompt to tune. All you can
change is how the question is put. So change it, and measure.
"""
import sys

import kev
from tickets import TICKETS

SITUATION = "A support desk is reading a customer message:\n{ticket}"

WORDINGS = {
    "the first attempt":
        "Is this customer asking for a refund?",
    "say what counts as yes":
        "Is this customer asking us to give them money back, in any words?",
    "rule the near miss out":
        "Is this customer asking us to return money they already paid? "
        "Asking about the policy does not count.",
}

sys.stdout.reconfigure(encoding="utf-8")


def score(ask_text):
    """Run every ticket through one wording and count the hits."""
    right, wrong = 0, []
    for ticket, wanted in TICKETS:
        sure = kev.noul(SITUATION.format(ticket=ticket), ask_text)
        if (sure >= 0.5) == wanted:
            right = right + 1
        else:
            wrong.append((sure, wanted, ticket))
    return right, wrong


def main():
    kev.session()
    for label, ask_text in WORDINGS.items():
        right, wrong = score(ask_text)
        print(f"{label:26} {right:2}/{len(TICKETS)}")
        for sure, wanted, ticket in wrong:
            print(f"     {sure:5.3f} wanted {str(wanted):5} {ticket[:44]}")
        print()


if __name__ == "__main__":
    main()

the first attempt          10/15
say what counts as yes     12/15
rule the near miss out      6/15

"""gate1.py — challenge 2: let the fast model answer what it is sure of.

Kev is about five times faster and gets fewer right. Instead of choosing
one, use the number: act on Kev where it is confident, and spend the slow
model only on the middle.
"""
import json
import sys
import time
import urllib.request

import kev
from tickets import TICKETS

OLLAMA = "http://localhost:11434/api/chat"
MODEL = "llama3.2"

SITUATION = "A support desk is reading a customer message:\n{ticket}"
ASK = "Is this customer asking us to give them money back, in any words?"
PROMPT = (SITUATION + "\n\n" + ASK + "\nAnswer with JSON only:\n"
          '{{"refund": true or false}}')

sys.stdout.reconfigure(encoding="utf-8")


def ask_llm(ticket):
    body = json.dumps({"model": MODEL, "stream": False, "format": "json",
                       "options": {"temperature": 0},
                       "messages": [{"role": "user",
                                     "content": PROMPT.format(ticket=ticket)}]}).encode()
    request = urllib.request.Request(OLLAMA, data=body,
                                     headers={"Content-Type": "application/json"})
    with urllib.request.urlopen(request, timeout=300) as response:
        answer = json.loads(response.read())
    return bool(json.loads(answer["message"]["content"]).get("refund", False))


def main():
    kev.session()

    # Score everything once with both, then try bands without paying again.
    scored = []
    kev_total, llm_total = 0.0, 0.0
    for ticket, wanted in TICKETS:
        start = time.perf_counter()
        sure = kev.noul(SITUATION.format(ticket=ticket), ASK)
        kev_total += time.perf_counter() - start

        start = time.perf_counter()
        said = ask_llm(ticket)
        llm_total += time.perf_counter() - start
        scored.append((sure, said, wanted))

    kev_ms = kev_total / len(TICKETS) * 1000
    llm_ms = llm_total / len(TICKETS) * 1000
    print(f"kev alone       {sum((s >= 0.5) == w for s, _, w in scored)}"
          f"/{len(scored)}   {kev_ms:6.0f} ms each")
    print(f"llama3.2 alone  {sum(d == w for _, d, w in scored)}"
          f"/{len(scored)}   {llm_ms:6.0f} ms each")
    print()

    print(f"{'band':>14} {'right':>7} {'asked the llm':>15} {'mean ms':>9}")
    for low, high in [(0.5, 0.5), (0.3, 0.7), (0.2, 0.8), (0.1, 0.9), (0.05, 0.95)]:
        right, sent = 0, 0
        for sure, said, wanted in scored:
            if sure <= low or sure >= high:
                call = sure >= 0.5
            else:
                call = said
                sent = sent + 1
            right += int(call == wanted)
        mean = kev_ms + (sent / len(scored)) * llm_ms
        print(f"  {low:.2f}-{high:.2f} {right:8}/{len(scored)} "
              f"{sent:10}/{len(scored)} {mean:9.0f}")


if __name__ == "__main__":
    main()

kev alone       12/15      447 ms each
llama3.2 alone  15/15     1006 ms each

          band   right   asked the llm   mean ms
  0.50-0.50       12/15          0/15       447
  0.30-0.70       13/15          3/15       648
  0.20-0.80       13/15          5/15       783
  0.10-0.90       14/15          7/15       917
  0.05-0.95       14/15         10/15      1118

"""drift1.py — challenge 3: the same model, two machines, different numbers.

Run this on one computer, then on another, then run it again on either.
It compares the two files. A 4-bit model does its arithmetic slightly
differently on different processors, and near the middle that changes
the answer.
"""
import json
import platform
import sys
from pathlib import Path

import kev
from tickets import TICKETS

SITUATION = "A support desk is reading a customer message:\n{ticket}"
ASK = "Is this customer asking us to give them money back, in any words?"

HERE = Path(__file__).parent
MINE = HERE / ("scores-" + platform.machine().lower() + ".json")

sys.stdout.reconfigure(encoding="utf-8")


def main():
    kev.session()
    scores = [kev.noul(SITUATION.format(ticket=t), ASK) for t, _ in TICKETS]
    MINE.write_text(json.dumps(scores), encoding="utf-8")
    print(f"wrote {MINE.name} on {platform.machine()}")

    others = sorted(p for p in HERE.glob("scores-*.json") if p != MINE)
    if not others:
        print("run it on another machine, copy the file here, run it again")
        return

    other = others[0]
    theirs = json.loads(other.read_text(encoding="utf-8"))
    print(f"comparing with {other.name}")
    print()

    flips, biggest = 0, 0.0
    for (ticket, _), mine, theirsb in zip(TICKETS, scores, theirs):
        gap = abs(mine - theirsb)
        biggest = max(biggest, gap)
        if (mine >= 0.5) != (theirsb >= 0.5):
            flips = flips + 1
            print(f"  FLIPPED  {mine:5.3f} -> {theirsb:5.3f}  {ticket[:42]}")
        elif gap > 0.05:
            print(f"           {mine:5.3f} -> {theirsb:5.3f}  {ticket[:42]}")

    print()
    print(f"{flips} of {len(TICKETS)} answers changed side")
    print(f"biggest difference in the numbers: {biggest:.3f}")


if __name__ == "__main__":
    main()

comparing with scores-aarch64.json

           0.668 -> 0.594  I was charged twice for the same order
           0.211 -> 0.287  Can you send me an invoice for last month
           0.476 -> 0.408  Do you offer a student discount before I
           0.239 -> 0.356  What is your refund policy, out of interest

0 of 15 answers changed side
biggest difference in the numbers: 0.118