# score.py — how many messages does a free System 1 model send to the right helper?
#
#   pip install openthai-systemone
#   python score.py pick_questions.json
#
# The file holds the question put to the model, what each helper is for, and the
# messages, each with the helper a person chose for it BEFORE any model was run.
# To test your own messages, put them in the file, and write each "want" first.
import json
import sys
from collections import Counter

from openthai_systemone import Choice, SystemOneClient

asked = json.load(open(sys.argv[1], encoding="utf-8"))
model = SystemOneClient("iapp/OpenThai-SystemOne", device="cpu")
question = {"tool": Choice(instructions=asked["instructions"], criteria=asked["criteria"])}

right, of, missed = Counter(), Counter(), []
for q in asked["questions"]:
    picked = model.system_one(state=q["text"], questions=question).answers["tool"].choice
    language = q.get("lang", "all")
    of[language] += 1
    if picked == q["want"]:
        right[language] += 1
    else:
        missed.append(f'  wanted {q["want"]}, picked {picked}: {q["text"]}')

print(f"right: {sum(right.values())} of {sum(of.values())}")
for language in of:
    print(f"  {language}: {right[language]} of {of[language]}")
print("missed:")
print("\n".join(missed))