🎯 Decider: a model that picks, and says how sure it is

1One question, three options

📄 NEW FILE · first.py

"""first.py - load the model, ask one question, and print what comes back.

    pip install decider-ai
    python first.py

The model runs on this computer. The first run downloads it: 3.8 GB.
"""
from decider.infer import Decider

MESSAGE = ("The air conditioner in room 12 has been dripping water onto the bed "
           "since last night. Please send someone before 2 pm. I fly home tomorrow.")

model = Decider("Mapika/decider-2b")

answers = model.decide(MESSAGE, [
    {"question": "Which team should handle this message?",
     "options": ["housekeeping", "maintenance", "front desk"]},
])

answer = answers[0]            # decide() gives back a list: one answer for each question
print("choice:", answer["choice"])
print("confidence:", round(answer["confidence"], 4))
for option, probability in answer["probs"].items():
    print(f"  {probability:.4f}  {option}")

🖥️ WHAT IT PRINTED

choice: housekeeping
confidence: 0.4887
  0.4887  housekeeping
  0.3596  maintenance
  0.1516  front desk

2The same answer, by hand

📄 NEW FILE · by_hand.py

"""by_hand.py - the same answer without the decider library: one pass, three letters.

    python by_hand.py

It writes the text the model reads, runs the model once, and looks at the
numbers the model gives the letters A, B and C. No text is generated.
"""
import torch
from transformers import AutoModelForCausalLM, AutoTokenizer

MESSAGE = ("The air conditioner in room 12 has been dripping water onto the bed "
           "since last night. Please send someone before 2 pm. I fly home tomorrow.")
OPTIONS = ["housekeeping", "maintenance", "front desk"]
# The model's setting for a choice question. It is in its decider_config.json.
TEMPERATURE = 1.164

# 1. The text the model reads. It stops where the letter of the answer would come.
prompt = "Context:\n" + MESSAGE
prompt += "\n\nQuestion: Which team should handle this message?\nOptions:"
for letter, option in zip("ABC", OPTIONS):
    prompt += f"\n({letter}) {option}"
prompt += "\nAnswer: ("
print(prompt)

# 2. One pass. For every token it knows, the model gives a logit:
#    a number that is high when that token is likely to come next.
#    float32 is the kind of number the decider library uses on a CPU.
tokenizer = AutoTokenizer.from_pretrained("Mapika/decider-2b")
model = AutoModelForCausalLM.from_pretrained("Mapika/decider-2b", dtype=torch.float32)
tokens = tokenizer(prompt, return_tensors="pt")
with torch.no_grad():
    next_logits = model(**tokens).logits[0, -1]

top_ids = torch.topk(next_logits, 5).indices.tolist()
read = len(tokens["input_ids"][0])
print()
print(read, "tokens were read;", len(next_logits), "tokens have a logit")
print("the five highest:", [tokenizer.decode([token_id]) for token_id in top_ids])

# 3. Keep the logits of A, B and C. Divide by the temperature. Turn them into shares of 1.
letter_ids = [tokenizer.encode(letter, add_special_tokens=False)[0] for letter in "ABC"]
logits = next_logits[letter_ids]
probabilities = torch.softmax(logits / TEMPERATURE, dim=-1)
rows = zip("ABC", OPTIONS, logits.tolist(), probabilities.tolist())
for letter, option, logit, probability in rows:
    print(f"({letter})  logit {logit:.4f}  probability {probability:.4f}  {option}")

🖥️ WHAT IT PRINTED · the text the model read, then the numbers

Context:
The air conditioner in room 12 has been dripping water onto the bed since last night. Please send someone before 2 pm. I fly home tomorrow.

Question: Which team should handle this message?
Options:
(A) housekeeping
(B) maintenance
(C) front desk
Answer: (

66 tokens were read; 248320 tokens have a logit
the five highest: ['A', 'B', 'C', 'D', 'F']
(A)  logit 13.5235  probability 0.4887  housekeeping
(B)  logit 13.1665  probability 0.3596  maintenance
(C)  logit 12.1614  probability 0.1516  front desk

1. 1 · The textthe message, the question, the options, then Answer: (
2. 2 · One passthe model reads all the tokens once
3. 3 · Three logitsthe numbers for A, B and C
4. 4 · Divideeach one by the temperature
5. 5 · Softmaxthe three become shares of 1

3choice: one of several

📄 NEW FILE · choice.py

"""choice.py - a choice question: one of several options, each with its meaning.

    python choice.py
"""
import json

from decider.infer import Decider

MESSAGE = ("The air conditioner in room 12 has been dripping water onto the bed "
           "since last night. Please send someone before 2 pm. I fly home tomorrow.")

model = Decider("Mapika/decider-2b")

question = {
    "type": "choice",
    "instructions": "Which team should handle this message?",
    "criteria": {                                  # option name: what the option means
        "housekeeping": "Cleaning, towels, bed linen",
        "maintenance": "Repairs: air conditioner, water, electricity",
        "front_desk": "Bookings, check-out, bills, questions",
        "other": None,                             # None: the name alone
    },
}

result = model.system_one(MESSAGE, {"team": question})
print(json.dumps(result, indent=2))

🖥️ WHAT IT PRINTED

{
  "model": "decider-2b-v11",
  "answers": {
    "team": {
      "type": "choice",
      "choice": "maintenance",
      "confidence": 0.8022,
      "x_p_max": 0.8516,
      "certainty": 0.6104,
      "probabilities": {
        "housekeeping": 0.1023,
        "maintenance": 0.8516,
        "front_desk": 0.0321,
        "other": 0.014
      }
    }
  },
  "usage": {
    "input_tokens": 98,
    "output_tokens": 0
  }
}

4noul: yes or no

📄 NEW FILE · noul.py

"""noul.py - a yes/no question. The answer is one number: how much "yes".

    python noul.py
"""
import json

from decider.infer import Decider

MESSAGE = ("The air conditioner in room 12 has been dripping water onto the bed "
           "since last night. Please send someone before 2 pm. I fly home tomorrow.")

model = Decider("Mapika/decider-2b")

questions = {
    # the usual form: the question as a sentence
    "refund": {"type": "noul", "instructions": "Does the guest ask for money back?"},
    # the other form: no sentence, but what "yes" and "no" mean
    "today": {"type": "noul", "criteria": {"true": "The guest needs help today",
                                           "false": "It can wait"}},
}

result = model.system_one(MESSAGE, questions)
print(json.dumps(result["answers"], indent=2))

🖥️ WHAT IT PRINTED

{
  "refund": {
    "type": "noul",
    "noul": 0.0304
  },
  "today": {
    "type": "noul",
    "noul": 0.8425
  }
}

5score: where on a scale

📄 NEW FILE · score.py

"""score.py - a score question: where on a scale? The levels are written out, low to high.

    python score.py
"""
import json

from decider.infer import Decider

MESSAGE = ("The air conditioner in room 12 has been dripping water onto the bed "
           "since last night. Please send someone before 2 pm. I fly home tomorrow.")

model = Decider("Mapika/decider-2b")

question = {
    "type": "score",
    "instructions": "How soon does the guesthouse have to act?",
    "criteria": [                                   # level 0, level 1, level 2
        "It can wait a few days",
        "It should be done today",
        "It must be done within a few hours",
    ],
}

result = model.system_one(MESSAGE, {"urgent": question})
print(json.dumps(result["answers"], indent=2))

🖥️ WHAT IT PRINTED

{
  "urgent": {
    "type": "score",
    "score": 1.57,
    "confidence": 0.3618,
    "x_p_max": 0.5819,
    "certainty": 0.3474,
    "legend": {
      "0": "It can wait a few days",
      "1": "It should be done today",
      "2": "It must be done within a few hours"
    },
    "probabilities": {
      "0": 0.0074,
      "1": 0.4106,
      "2": 0.5819
    },
    "level_fit": {
      "0": 0.0063,
      "1": 0.3473,
      "2": 0.4922
    },
    "fit_mass": 0.8457
  }
}

6A record, and three questions in one call

📄 NEW FILE · one_call.py

"""one_call.py - a record as the state, and three questions in one call.

    python one_call.py
"""
from decider.infer import Decider

MESSAGE = ("The air conditioner in room 12 has been dripping water onto the bed "
           "since last night. Please send someone before 2 pm. I fly home tomorrow.")

# The state does not have to be a sentence. A dictionary or a list works too.
BOOKING = {
    "guest": {"name": "Anna", "room": 12, "nights": 3},
    "messages": [{"from": "guest", "text": MESSAGE}],
}

# A question can name one part of the state by its path, between backticks.
QUESTIONS = {
    "team": {
        "type": "choice",
        "instructions": "Which team should handle `messages[0].text`?",
        "criteria": {
            "housekeeping": "Cleaning, towels, bed linen",
            "maintenance": "Repairs: air conditioner, water, electricity",
            "front_desk": "Bookings, check-out, bills, questions",
            "other": None,
        },
    },
    "refund": {
        "type": "noul",
        "instructions": "Does `messages[0].text` ask for money back?",
    },
    "urgent": {
        "type": "score",
        "instructions": "How soon does the guesthouse have to act?",
        "criteria": ["It can wait a few days", "It should be done today",
                     "It must be done within a few hours"],
    },
}

model = Decider("Mapika/decider-2b")
result = model.system_one(BOOKING, QUESTIONS)

for name, answer in result["answers"].items():
    kind = answer["type"]                  # "choice", "noul" or "score"
    # The answer itself is under the key that has the name of its type.
    print(f"{name:7} {kind:7} {answer[kind]}")
print(result["usage"])

🖥️ WHAT IT PRINTED

team    choice  maintenance
refund  noul    0.0232
urgent  score   1.5
{'input_tokens': 299, 'output_tokens': 0}

7Ask the library what it takes

📄 NEW FILE · look_inside.py

"""look_inside.py - ask the library what its functions take. No model is loaded.

    python look_inside.py
"""
import inspect

import decider.systemone
from decider.infer import Decider

# A name with no brackets after it is the function itself, and Python can look at it.
# With brackets, Decider.system_one(), Python would call it.
print("decide", inspect.signature(Decider.decide))
print("decide_batch", inspect.signature(Decider.decide_batch))
print("decide_json", inspect.signature(Decider.decide_json))
print("system_one", inspect.signature(Decider.system_one))
print()

# The library has no type hints. What a question looks like is written in this docstring.
print(decider.systemone.__doc__)

🖥️ WHAT IT PRINTED

decide (self, context, questions, **kw)
decide_batch (self, requests, max_ctx_tokens=1536)
decide_json (self, context, schema, **kw)
system_one (self, state, questions, independent=True, max_state_tokens=32768, max_fwd_tokens=65536, layout=None, isolated=None)

Jev-shaped requests on top of the decider prompt format (same wire format as TypeSafe's POST /v1/systemone).

    state      str | dict | list            JSON state is serialised compactly; questions may name a part by path (`ticket.messages[0].text`)
    questions  {id: {"type": "choice", "instructions": ..., "criteria": {name: description | {...} | [...] | None}}      up to 255 options
                     {"type": "score",  "instructions": ..., "criteria": [level 0 description, level 1 description, ...]}  2..10 levels
                     {"type": "noul",   "instructions": ... (optional), "criteria": {"true": ..., "false": ...} (optional)}}
    a noul question needs instructions or at least one true/false description.
    ids are never shown to the model.  `instructions` and every description may be a string or any JSON value.

8decide_json(): the question is the key

📄 NEW FILE · fields.py

"""fields.py - decide_json(): the question is the key, its field is the kind of answer.

    python fields.py
"""
import json

from decider.infer import Decider

MESSAGE = ("The air conditioner in room 12 has been dripping water onto the bed "
           "since last night. Please send someone before 2 pm. I fly home tomorrow.")

FIELDS = {
    "Which team should handle this message?": {
        "type": "choice",
        "options": ["housekeeping: cleaning, towels, bed linen",
                    "maintenance: repairs, air conditioner, water, electricity",
                    "front desk: bookings, check-out, bills, questions"],
    },
    "Does the guest ask for money back?": {"type": "bool"},
    "How soon does the guesthouse have to act?": {
        "type": "scale",
        "legend": ["it can wait a few days", "it should be done today",
                   "it must be done within a few hours"],
    },
}

model = Decider("Mapika/decider-2b")
answers = model.decide_json(MESSAGE, FIELDS)
print(json.dumps(answers, indent=2))

🖥️ WHAT IT PRINTED

{
  "Which team should handle this message?": {
    "choice": "maintenance: repairs, air conditioner, water, electricity",
    "confidence": 0.8762,
    "type": "choice",
    "probabilities": {
      "housekeeping: cleaning, towels, bed linen": 0.1014,
      "maintenance: repairs, air conditioner, water, electricity": 0.8762,
      "front desk: bookings, check-out, bills, questions": 0.0225
    }
  },
  "Does the guest ask for money back?": {
    "noul": 0.016,
    "type": "noul"
  },
  "How soon does the guesthouse have to act?": {
    "score": 1.89,
    "confidence": 0.8973,
    "type": "scale",
    "legend": [
      "it can wait a few days",
      "it should be done today",
      "it must be done within a few hours"
    ],
    "probabilities": {
      "0": 0.0034,
      "1": 0.0993,
      "2": 0.8973
    }
  }
}

9decide_batch(): many texts at once

📄 NEW FILE · many.py

"""many.py - six messages in one call, and no answer where the model is not sure.

    python many.py
"""
from decider.infer import Decider

MESSAGES = [
    "Can we have two more towels, please?",
    "The wifi password on the card does not work.",
    "What time is breakfast?",
    "My bill shows three nights but I stayed two.",
    "The shower has no hot water.",
    "There is a gecko on my wall and I love it.",
]
QUESTION = {
    "question": "Which team should handle this message?",
    "options": ["housekeeping: cleaning, towels, bed linen",
                "maintenance: repairs, air conditioner, water, electricity, wifi",
                "front desk: bookings, bills, questions"],
}

# abstain_below: when the best option has less than this, the choice is None.
model = Decider("Mapika/decider-2b", abstain_below=0.7)

# One request for each message: the text, and the list of its questions.
results = model.decide_batch([(message, [QUESTION]) for message in MESSAGES])

for message, answers in zip(MESSAGES, results):
    answer = answers[0]
    team = answer["choice"].split(":")[0] if answer["choice"] else "(not sure)"
    print(f"{answer['confidence']:.3f}  {team:13}  {message}")

🖥️ WHAT IT PRINTED

0.934  housekeeping   Can we have two more towels, please?
0.888  maintenance    The wifi password on the card does not work.
0.955  front desk     What time is breakfast?
0.991  front desk     My bill shows three nights but I stayed two.
0.932  maintenance    The shower has no hot water.
0.420  (not sure)     There is a gecko on my wall and I love it.

10It cannot compare numbers

📄 NEW FILE · compare.py

"""compare.py - can the model compare two numbers? And what a program does about it.

    python compare.py

Twelve river levels beside a bank of 3.7 m. First the model gets the two numbers.
Then the program compares them and writes the result as words.
"""
from decider.infer import Decider

BANK = 3.7
LEVELS = [3.2, 3.9, 2.1, 3.75, 5.3, 3.65, 4.0, 1.2, 3.71, 3.0, 6.8, 3.69]
ASK = "Is the river higher than the bank?"
QUESTION = {"over": {"type": "noul", "instructions": ASK}}


def yes(model, text):
    """How much "yes" the model gives the question for this text."""
    return model.system_one(text, QUESTION)["answers"]["over"]["noul"]


def main():
    model = Decider("Mapika/decider-2b")
    right_from_numbers = right_from_words = 0
    print("river  truth  numbers only   with the words")
    for level in LEVELS:
        numbers = f"The river is at {level} m. The river bank is {BANK} m high."
        # The program compares the two numbers. The model only reads the result.
        compared = "higher than" if level > BANK else "lower than"
        words = numbers + f" The river is {compared} the bank."
        truth = level > BANK
        from_numbers, from_words = yes(model, numbers), yes(model, words)
        right_from_numbers += (from_numbers > 0.5) == truth
        right_from_words += (from_words > 0.5) == truth
        answer = "yes" if truth else "no"
        print(f"{level:5}  {answer:5}  {from_numbers:.4f}         {from_words:.4f}")
    print(f"right with the numbers only: {right_from_numbers} of {len(LEVELS)}")
    print(f"right with the words: {right_from_words} of {len(LEVELS)}")


if __name__ == "__main__":
    main()

🖥️ WHAT IT PRINTED · the two right-hand columns are the probability of yes

river  truth  numbers only   with the words
  3.2  no     0.1014         0.0248
  3.9  yes    0.2579         0.8156
  2.1  no     0.0940         0.0242
 3.75  yes    0.2026         0.7521
  5.3  yes    0.1957         0.8179
 3.65  no     0.1244         0.0255
  4.0  yes    0.2168         0.8219
  1.2  no     0.0875         0.0258
 3.71  yes    0.1718         0.7286
  3.0  no     0.1095         0.0275
  6.8  yes    0.2685         0.8363
 3.69  no     0.1375         0.0261
right with the numbers only: 6 of 12
right with the words: 12 of 12

11Extra text moves the answer

📄 NEW FILE · too_much.py

"""too_much.py - text on the same subject changes an answer, wherever the question points.

    python too_much.py

One yes/no question about a guest's message, asked with more and more around the message.
"""
from decider.infer import Decider

MESSAGE = ("The air conditioner in room 12 has been dripping water onto the bed "
           "since last night. Please send someone before 2 pm. I fly home tomorrow.")
BOOKING = {
    "guest": {"name": "Anna", "room": 12, "nights": 3},
    "messages": [{"from": "guest", "text": MESSAGE}],
}
POLICY = "A night in a room that cannot be slept in is refunded."

ASK = "Does the guest ask for money back?"
ASK_BY_PATH = "Does `messages[0].text` ask for money back?"


def yes(model, state, instructions):
    """How much "yes" the model gives this question about this state."""
    question = {"refund": {"type": "noul", "instructions": instructions}}
    return model.system_one(state, question)["answers"]["refund"]["noul"]


def main():
    model = Decider("Mapika/decider-2b")
    # The same record, with the hotel's rule about refunds added to it.
    with_policy = dict(BOOKING, policy=POLICY)
    rows = [
        (MESSAGE, ASK, "the message alone"),
        (BOOKING, ASK, "the booking record"),
        (with_policy, ASK, "the record with the refund rule in it"),
        (with_policy, ASK_BY_PATH, "the same, and the question names the message"),
    ]
    for state, instructions, label in rows:
        print(f"{yes(model, state, instructions):.4f}  {label}")


if __name__ == "__main__":
    main()

🖥️ WHAT IT PRINTED · the probability of yes

0.0304  the message alone
0.0196  the booking record
0.6109  the record with the refund rule in it
0.5281  the same, and the question names the message

12A long text is cut

📄 NEW FILE · long_text.py

"""long_text.py - decide() reads only the start of a long text.

    python long_text.py

The text is a long list of house rules, and then the guest's message at the end.
"""
from decider.infer import Decider
from transformers import AutoTokenizer

MESSAGE = ("The air conditioner in room 12 has been dripping water onto the bed "
           "since last night. Please send someone before 2 pm. I fly home tomorrow.")
RULE = "Rule {number}: the guest in room {number} returns the key before noon."
ASK = "Which team should handle the guest's message?"
TEAMS = {"housekeeping": "cleaning, towels, bed linen",
         "maintenance": "repairs, air conditioner, water, electricity",
         "front desk": "bookings, check-out, bills, questions"}

# decide() takes each option as one string: "name: what it is for".
QUESTION = {"question": ASK,
            "options": [f"{team}: {work}" for team, work in TEAMS.items()]}


def show(label, answer):
    team = answer["choice"].split(":")[0]
    print(f"{answer['confidence']:.4f}  {team:12}  {label}")


def main():
    rules = "\n".join(RULE.format(number=number) for number in range(1, 151))
    text = rules + "\n\nGuest message: " + MESSAGE
    tokenizer = AutoTokenizer.from_pretrained("Mapika/decider-2b")
    print(len(tokenizer.encode(text)), "tokens in the text")

    model = Decider("Mapika/decider-2b")
    show("decide(): it reads the first 1,536 tokens",
         model.decide(text, [QUESTION])[0])
    show("decide() with max_ctx_tokens=8192",
         model.decide(text, [QUESTION], max_ctx_tokens=8192)[0])
    show("decide(), with the message first and the rules after it",
         model.decide("Guest message: " + MESSAGE + "\n\n" + rules, [QUESTION])[0])

    # system_one() reads up to 32,768 tokens, so it needs no extra setting here.
    question = {"team": {"type": "choice", "instructions": ASK, "criteria": TEAMS}}
    answer = model.system_one(text, question)["answers"]["team"]
    label = "system_one(), the same long text"
    print(f"{answer['x_p_max']:.4f}  {answer['choice']:12}  {label}")


if __name__ == "__main__":
    main()

🖥️ WHAT IT PRINTED

2969 tokens in the text
0.9852  front desk    decide(): it reads the first 1,536 tokens
0.6237  maintenance   decide() with max_ctx_tokens=8192
0.8349  maintenance   decide(), with the message first and the rules after it
0.6237  maintenance   system_one(), the same long text

13How fast, how big

📄 NEW FILE · speed.py

"""speed.py - how long the model takes on this computer.

    python speed.py

Each question is asked three times and the middle time is printed. The first
question after loading is slower, so one is asked and thrown away first.
"""
import time

from decider.infer import Decider

MESSAGE = ("The air conditioner in room 12 has been dripping water onto the bed "
           "since last night. Please send someone before 2 pm. I fly home tomorrow.")
CHOICE = {"type": "choice", "instructions": "Which team should handle this message?",
          "criteria": {"housekeeping": "Cleaning, towels, bed linen",
                       "maintenance": "Repairs: air conditioner, water, electricity",
                       "front_desk": "Bookings, check-out, bills, questions",
                       "other": None}}
NOUL = {"type": "noul", "instructions": "Does the guest ask for money back?"}
SCORE = {"type": "score", "instructions": "How soon does the guesthouse have to act?",
         "criteria": ["It can wait a few days", "It should be done today",
                      "It must be done within a few hours"]}
SHORT = ["Can we have two more towels, please?", "What time is breakfast?",
         "The shower has no hot water.", "The wifi password does not work.",
         "My bill shows three nights but I stayed two.", "Is there a laundry service?"]
PLAIN = {"question": "Which team should handle this message?",
         "options": ["housekeeping", "maintenance", "front desk"]}


def middle(times):
    return sorted(times)[len(times) // 2]


def ask(model, questions):
    """Seconds for one system_one() call with these questions: the middle of three."""
    times = []
    for _ in range(3):
        started = time.perf_counter()
        model.system_one(MESSAGE, questions)
        times.append(time.perf_counter() - started)
    return middle(times)


def main():
    started = time.perf_counter()
    model = Decider("Mapika/decider-2b")
    print(f"load the model                     {time.perf_counter() - started:5.1f} s")
    model.system_one(MESSAGE, {"first": NOUL})                  # thrown away
    print(f"one yes/no                         {ask(model, {'q': NOUL}):5.1f} s")
    print(f"one choice, four options           {ask(model, {'q': CHOICE}):5.1f} s")
    print(f"one score, three levels            {ask(model, {'q': SCORE}):5.1f} s")
    all_three = {"a": CHOICE, "b": NOUL, "c": SCORE}
    print(f"all three in one call              {ask(model, all_three):5.1f} s")
    times = []
    for _ in range(3):
        started = time.perf_counter()
        model.decide_batch([(message, [PLAIN]) for message in SHORT])
        times.append(time.perf_counter() - started)
    print(f"six short messages, decide_batch   {middle(times):5.1f} s")


if __name__ == "__main__":
    main()

🖥️ WHAT IT PRINTED · a laptop with no GPU

load the model                       6.0 s
one yes/no                           1.8 s
one choice, four options             3.3 s
one score, three levels              7.1 s
all three in one call                7.4 s
six short messages, decide_batch     5.0 s

🖥️ ALL THIRTEEN FILES · the seconds, and the most memory held

python 3.12.10 on Windows AMD64, 22 logical cores; decider-ai 1.8.1, torch 2.11.0, transformers 5.8.0
first.py         13.7 s  10.4 GB
by_hand.py       13.0 s   9.8 GB
choice.py        14.5 s  10.3 GB
noul.py          15.8 s  11.0 GB
score.py         16.9 s  10.3 GB
one_call.py      22.4 s  10.2 GB
look_inside.py    6.1 s   0.4 GB
fields.py        16.6 s  10.9 GB
many.py          20.7 s  10.1 GB
compare.py       42.4 s  10.7 GB
too_much.py      22.5 s  10.6 GB
long_text.py    148.4 s  10.4 GB
speed.py         89.0 s   9.6 GB