🔢 Meaning, as a row of numbers · ความหมาย ในรูปแถวของตัวเลข · 意思，就是一排数字

1 Three lines, and it has an opinion · สามบรรทัด แล้วมันก็มีความเห็น · 三行，它就有看法了

📄 NEW FILE · feel1.py

"""feel1.py — a real model, on your own computer, in three lines."""
from transformers import pipeline

# pipeline downloads a model the first time and keeps it. After that it is
# a file on your disk and none of this needs the internet.
#   text-classification  - the job you want done
# Leave the model out and it picks a small English one for you.
judge = pipeline("text-classification")

LINES = ["The khao soi here is the best in Chiang Mai.",
         "The bus was two hours late and nobody said anything.",
         "It is raining."]


def main():
    """Ask the model how each line feels, and print what it said."""
    for line in LINES:
        # One line in, a list with one answer out. The answer is a dict
        # with a label and a score between 0 and 1.
        answer = judge(line)[0]
        print(f"{answer['label']:8} {answer['score']:.3f}  {line}")


if __name__ == "__main__":
    main()

🖥️ WHAT IT PRINTED · three sentences, two labels

POSITIVE 1.000  The khao soi here is the best in Chiang Mai.
NEGATIVE 0.998  The bus was two hours late and nobody said anything.
POSITIVE 0.991  It is raining.

2 What the model actually reads · สิ่งที่โมเดลอ่านจริง ๆ · 模型真正读到的东西

📄 NEW FILE · tokens1.py

"""tokens1.py — what the model actually reads, which is not letters."""
from transformers import AutoTokenizer

# Two tokenizers, because the answer is different for each. The first was
# trained on English. The second saw a hundred languages.
ENGLISH = "sentence-transformers/all-MiniLM-L6-v2"
MANY = "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2"

LINES = ["read my stars",
         "ขอดวงประจำวัน",
         "今天的运势如何"]


def show(name, text):
    """Split one line the way one model would, and print the pieces."""
    tokenizer = AutoTokenizer.from_pretrained(name)

    # A model never sees letters. It sees numbers, one per token, and a
    # token is whatever its tokenizer decided a useful piece was.
    pieces = tokenizer.tokenize(text)
    numbers = tokenizer.encode(text, add_special_tokens=False)
    print(f"  {len(pieces):3} pieces  {pieces}")
    print(f"      as numbers  {numbers}")


def main():
    for text in LINES:
        print()
        print(text)
        print("  -- a tokenizer trained on English --")
        show(ENGLISH, text)
        print("  -- a tokenizer that saw a hundred languages --")
        show(MANY, text)


if __name__ == "__main__":
    main()

🖥️ WHAT IT PRINTED · the same three lines, two tokenizers

read my stars
  english      3 pieces  ['read', 'my', 'stars']
  multilingual 4 pieces  ['▁read', '▁my', '▁star', 's']

ขอดวงประจำวัน
  english      1 pieces  ['[UNK]']
  multilingual 3 pieces  ['▁ขอ', 'ดวง', 'ประจําวัน']

今天的运势如何
  english      7 pieces  ['[UNK]', '天', '的', '[UNK]', '[UNK]', '[UNK]', '[UNK]']
  multilingual 5 pieces  ['▁', '今天的', '运', '势', '如何']

3 Four sentences, and how close they are · สี่ประโยค กับความใกล้ของมัน · 四句话，和它们有多近

📄 NEW FILE · embed1.py

"""embed1.py — a sentence, turned into numbers you can compare."""
from sentence_transformers import SentenceTransformer

# A small model, about 90 MB. It is downloaded once and kept.
model = SentenceTransformer("all-MiniLM-L6-v2")

LINES = ["I would like a horoscope please",
         "read my stars",
         "what do the stars say today",
         "your package has shipped"]


def main():
    """Turn four sentences into numbers, then compare them."""
    # encode gives one row of numbers per sentence. Sentences that mean
    # similar things end up with similar numbers.
    vectors = model.encode(LINES)
    print("shape:", vectors.shape)
    print("each sentence is", model.get_embedding_dimension(), "numbers")
    print("the first eight of the first one:", vectors[0][:8].round(3))

    print()
    print("how close is each line to the first one:")

    # similarity compares rows. 1.0 is the same sentence, 0 is unrelated.
    scores = model.similarity(vectors[0:1], vectors)[0]
    for line, score in zip(LINES, scores):
        bar = "#" * int(max(0, float(score)) * 40)
        print(f"  {float(score):6.3f} {bar:40} {line}")


if __name__ == "__main__":
    main()

🖥️ WHAT IT PRINTED · 384 numbers a sentence

shape: (4, 384)
the first eight of the first one: [-0.054  0.005 -0.002  0.058 -0.055 -0.032 -0.017 -0.024]

how close is each line to the first one:
   1.000  I would like a horoscope please
   0.418  read my stars
   0.327  what do the stars say today
   0.179  your package has shipped

4 The English model cannot read Thai · โมเดลภาษาอังกฤษอ่านภาษาไทยไม่ออก · 英文模型读不懂泰文

📄 NEW FILE · multi1.py

"""multi1.py — the same sentences, through two models. Only one can read Thai."""
from sentence_transformers import SentenceTransformer

ENGLISH = "all-MiniLM-L6-v2"
MANY = "paraphrase-multilingual-MiniLM-L12-v2"

LINES = ["I would like a horoscope please",
         "read my stars",
         "your package has shipped",
         "ขอดวงประจำวัน",          # may I have today's horoscope
         "ใบแจ้งหนี้ค่าไฟ",          # the electricity bill
         "今天的运势如何",           # how is today's fortune
         "您的快递已送达"]           # your parcel has arrived


def compare(name):
    """Score every line against the first one, and draw it."""
    model = SentenceTransformer(name)
    vectors = model.encode(LINES)
    scores = model.similarity(vectors[0:1], vectors)[0]

    print()
    print(name)
    for line, score in zip(LINES, scores):
        bar = "#" * int(max(0, float(score)) * 40)
        print(f"  {float(score):6.3f} {bar:40} {line}")


def main():
    compare(ENGLISH)
    compare(MANY)


if __name__ == "__main__":
    main()

🖥️ WHAT IT PRINTED · all-MiniLM-L6-v2, the English one

   1.000  I would like a horoscope please
   0.418  read my stars
   0.179  your package has shipped
   0.119  ขอดวงประจำวัน        <- asking for a horoscope
   0.119  ใบแจ้งหนี้ค่าไฟ       <- the electricity bill
   0.284  今天的运势如何
   0.145  您的快递已送达

🖥️ AND THEN · paraphrase-multilingual-MiniLM-L12-v2

   1.000  I would like a horoscope please
   0.317  read my stars
   0.047  your package has shipped
   0.366  ขอดวงประจำวัน        <- now the highest but one
   0.106  ใบแจ้งหนี้ค่าไฟ       <- and clearly apart
   0.241  今天的运势如何
   0.106  您的快递已送达

5 Search by meaning, in any language · ค้นหาด้วยความหมาย ในภาษาไหนก็ได้ · 按意思搜索，什么语言都行

📄 NEW FILE · search1.py

"""search1.py — search by meaning, not by matching words."""
from sentence_transformers import SentenceTransformer

model = SentenceTransformer("paraphrase-multilingual-MiniLM-L12-v2")

# Pretend these are the pages on a website. Nine short descriptions.
PAGES = [
    "A bot that reads your mail once an hour and replies with a horoscope",
    "Checking the bitcoin price and placing real orders with Python",
    "Build a chat window with tkinter and a model that remembers",
    "Classes in Python, taught by building a window piece by piece",
    "Turn a Python function into a web page with Gradio",
    "Send email from Python, and read your inbox with IMAP",
    "Automate jobs with n8n, without writing much code",
    "Put a website on the internet with Astro and Vercel",
    "Give a model tools it can choose to call, with smolagents",
]

# Encoded once, when the program starts. Searching is then only arithmetic.
LIBRARY = model.encode(PAGES)

QUESTIONS = ["how do I make a window",
             "I want to sell things online",
             "ส่งอีเมลด้วย Python",
             "让模型自己调用工具"]


def find(question, how_many=3):
    """Return the closest pages, best first."""
    asked = model.encode([question])

    # One number per page: how close it is to the question.
    scores = model.similarity(asked, LIBRARY)[0]

    # sorted() on pairs sorts by the first item, so put the score first.
    ranked = sorted(zip(scores.tolist(), PAGES), reverse=True)
    return ranked[:how_many]


def main():
    for question in QUESTIONS:
        print()
        print("Q:", question)
        for score, page in find(question):
            print(f"   {score:.3f}  {page}")


if __name__ == "__main__":
    main()

🖥️ WHAT IT PRINTED · four questions, nine pages

Q: how do I make a window
   0.398  Classes in Python, taught by building a window piece by piece
   0.373  Build a chat window with tkinter and a model that remembers

Q: I want to sell things online
   0.366  Put a website on the internet with Astro and Vercel
   0.249  Checking the bitcoin price and placing real orders with Python

Q: ส่งอีเมลด้วย Python
   0.755  Send email from Python, and read your inbox with IMAP
   0.388  Turn a Python function into a web page with Gradio

Q: 让模型自己调用工具
   0.556  Give a model tools it can choose to call, with smolagents

6 Sorting into piles, with no training · แยกของเข้ากอง โดยไม่ต้องเทรน · 分堆，不用训练

📄 NEW FILE · sort1.py

"""sort1.py — sorting mail into two piles, with no training at all.

Write a few examples of each pile. Turn everything into numbers. Put each
new subject on whichever side it sits closer to. That is the whole method.
"""
from sentence_transformers import SentenceTransformer

model = SentenceTransformer("paraphrase-multilingual-MiniLM-L12-v2")

ASKING = ["can I have my horoscope",
          "what do the stars say about me today",
          "please send me a fortune reading",
          "tell me about my star sign"]

NOT_ASKING = ["your invoice is attached",
              "the meeting has been moved",
              "your parcel is on its way",
              "please stop sending me this newsletter"]

# Encoded once. Every subject after this is compared against these.
YES = model.encode(ASKING)
NO = model.encode(NOT_ASKING)

SUBJECTS = ["horoscope please",
            "read my stars",
            "ขอดวงประจำวัน",
            "今天的运势如何",
            "Your flight is confirmed",
            "my horoscope was wrong yesterday"]


def is_asking(subject):
    """Which pile is this subject closer to? Returns the answer and both scores."""
    vector = model.encode([subject])

    # max(), not mean(). One good match is enough, and averaging lets three
    # unrelated examples drown out the one that fits.
    near_yes = float(model.similarity(vector, YES).max())
    near_no = float(model.similarity(vector, NO).max())
    return near_yes > near_no, near_yes, near_no


def main():
    for subject in SUBJECTS:
        asking, yes, no = is_asking(subject)
        print(f"{'reply ' if asking else 'ignore'}  "
              f"yes={yes:.3f} no={no:.3f}   {subject}")


if __name__ == "__main__":
    main()

🖥️ WHAT IT PRINTED · five right, one badly wrong

reply   yes=0.870 no=0.140   horoscope please
reply   yes=0.686 no=0.207   read my stars
reply   yes=0.451 no=0.345   ขอดวงประจำวัน
reply   yes=0.507 no=0.304   今天的运势如何
ignore  yes=0.294 no=0.381   Your flight is confirmed
reply   yes=0.766 no=0.148   my horoscope was wrong yesterday

7 Try first. The answers are folded away. · ลองเองก่อน เฉลยพับซ่อนไว้ · 先自己试。答案折起来放着。

"""dupes.py — challenge 1: find the sentences that say the same thing."""
from sentence_transformers import SentenceTransformer

model = SentenceTransformer("paraphrase-multilingual-MiniLM-L12-v2")

NOTES = [
    "The wifi password is on the fridge",
    "Wifi code is stuck to the fridge door",
    "Bring an umbrella, it rains every afternoon",
    "Take an umbrella. Afternoon rain is guaranteed",
    "The market closes at six",
    "รหัสไวไฟติดอยู่ที่ตู้เย็น",
    "Please water the plants on Tuesday",
]
CLOSE_ENOUGH = 0.7


def find_pairs():
    """Every pair that scores above the line, closest first."""
    vectors = model.encode(NOTES)
    scores = model.similarity(vectors, vectors)

    found = []
    # range(row + 1, ...) so each pair is looked at once, and never against
    # itself. Comparing a sentence to itself always gives 1.0.
    for row in range(len(NOTES)):
        for column in range(row + 1, len(NOTES)):
            score = float(scores[row][column])
            if score >= CLOSE_ENOUGH:
                found.append((score, NOTES[row], NOTES[column]))
    return sorted(found, reverse=True)


def main():
    print(len(NOTES), "notes, anything scoring", CLOSE_ENOUGH, "or more:")
    print()
    for score, one, two in find_pairs():
        print(f"  {score:.3f}")
        print(f"     {one}")
        print(f"     {two}")


if __name__ == "__main__":
    main()

0.848  Bring an umbrella, it rains every afternoon
       Take an umbrella. Afternoon rain is guaranteed
0.800  The wifi password is on the fridge
       Wifi code is stuck to the fridge door

"""better_sort.py — challenge 2: the traps, and what fixing them costs.

sort1.py got every trap wrong. Mail that is ABOUT horoscopes but is not
asking for one sits close to the asking examples, because embeddings
measure what a sentence is about. The fix is to show it the difference.
"""
from sentence_transformers import SentenceTransformer

model = SentenceTransformer("paraphrase-multilingual-MiniLM-L12-v2")

ASKING = ["can I have my horoscope",
          "what do the stars say about me today",
          "please send me a fortune reading",
          "tell me about my star sign"]

PLAIN_NO = ["your invoice is attached",
            "the meeting has been moved",
            "your parcel is on its way",
            "please stop sending me this newsletter"]

# The same four, plus four that are about horoscopes without asking for one.
# These are the shape sort1.py could not see.
BETTER_NO = PLAIN_NO + ["your horoscope yesterday was wrong",
                        "feedback on your astrology column",
                        "unsubscribe me from the horoscope newsletter",
                        "invoice for the horoscope app"]

TRAPS = ["my horoscope was wrong yesterday",
         "Horoscope column feedback",
         "Unsubscribe me from the Daily Horoscope newsletter",
         "Invoice for horoscope app subscription"]
REAL = ["horoscope please", "read my stars", "ขอดวงประจำวัน"]


def sorter(no_examples):
    """Build a sorter from one set of examples."""
    yes = model.encode(ASKING)
    no = model.encode(no_examples)

    def is_asking(subject):
        vector = model.encode([subject])
        return (float(model.similarity(vector, yes).max())
                > float(model.similarity(vector, no).max()))
    return is_asking


def main():
    plain = sorter(PLAIN_NO)
    better = sorter(BETTER_NO)

    print(f"{'subject':52} {'4 examples':>12} {'8 examples':>12}")
    print("-" * 78)
    for subject in TRAPS:
        print(f"{subject[:52]:52} {str(plain(subject)):>12} "
              f"{str(better(subject)):>12}   (should be False)")
    for subject in REAL:
        print(f"{subject[:52]:52} {str(plain(subject)):>12} "
              f"{str(better(subject)):>12}   (should be True)")


if __name__ == "__main__":
    main()

subject                                        4 ex    8 ex
my horoscope was wrong yesterday               True   False  (want False)
Horoscope column feedback                      True   False  (want False)
Unsubscribe me from the Daily Horoscope        True   False  (want False)
Invoice for horoscope app subscription         True   False  (want False)
horoscope please                               True    True  (want True)
read my stars                                  True    True  (want True)
ขอดวงประจำวัน                                  True    True  (want True)

"""rerank.py — challenge 3: a slower model, to check the fast one's work.

An embedding model reads the question and the page separately, so the
comparison is cheap but blind. A cross encoder reads both at once and
scores the pair. Too slow to search with, exactly right for checking the
handful the search brought back.
"""
from sentence_transformers import CrossEncoder, SentenceTransformer

fast = SentenceTransformer("paraphrase-multilingual-MiniLM-L12-v2")
careful = CrossEncoder("cross-encoder/ms-marco-MiniLM-L6-v2")

PAGES = [
    "A bot that reads your mail once an hour and replies with a horoscope",
    "Checking the bitcoin price and placing real orders with Python",
    "Build a chat window with tkinter and a model that remembers",
    "Classes in Python, taught by building a window piece by piece",
    "Turn a Python function into a web page with Gradio",
    "Send email from Python, and read your inbox with IMAP",
    "Automate jobs with n8n, without writing much code",
    "Put a website on the internet with Astro and Vercel",
    "Give a model tools it can choose to call, with smolagents",
]
LIBRARY = fast.encode(PAGES)


def search(question, keep=4):
    """The fast model picks a few. The slow one puts them in order."""
    scores = fast.similarity(fast.encode([question]), LIBRARY)[0]
    shortlist = sorted(zip(scores.tolist(), PAGES), reverse=True)[:keep]

    # The cross encoder wants pairs, and gives one number per pair.
    pairs = [(question, page) for _, page in shortlist]
    second = careful.predict(pairs)

    reordered = sorted(zip(second.tolist(), [p for _, p in shortlist]),
                       reverse=True)
    return shortlist, reordered


def main():
    for question in ["how do I put my site online",
                     "I want a program that answers emails"]:
        first, second = search(question)
        print()
        print("Q:", question)
        print("  the fast model's order:")
        for score, page in first:
            print(f"     {score:6.3f}  {page}")
        print("  after the cross encoder looked again:")
        for score, page in second:
            print(f"     {score:6.2f}  {page}")


if __name__ == "__main__":
    main()

Q: how do I put my site online
  fast model:   0.612  Astro and Vercel
                0.387  Gradio
                0.156  bitcoin
                0.092  tkinter
  cross encoder: -1.49  Astro and Vercel
                -10.81  Gradio
                -11.26  tkinter
                -11.41  bitcoin

pip install gradio

"""search_app.py — challenge 4: the search from step 5, as a web page."""
import gradio as gr
from sentence_transformers import SentenceTransformer

model = SentenceTransformer("paraphrase-multilingual-MiniLM-L12-v2")

PAGES = [
    "A bot that reads your mail once an hour and replies with a horoscope",
    "Checking the bitcoin price and placing real orders with Python",
    "Build a chat window with tkinter and a model that remembers",
    "Classes in Python, taught by building a window piece by piece",
    "Turn a Python function into a web page with Gradio",
    "Send email from Python, and read your inbox with IMAP",
    "Automate jobs with n8n, without writing much code",
    "Put a website on the internet with Astro and Vercel",
    "Give a model tools it can choose to call, with smolagents",
]

# Encoded once, when the program starts, not once per search. This is the
# whole reason a search like this is fast.
LIBRARY = model.encode(PAGES)


def find(question, how_many):
    """Return the best pages as rows, ready for a table."""
    if not question.strip():
        return []
    scores = model.similarity(model.encode([question]), LIBRARY)[0]
    ranked = sorted(zip(scores.tolist(), PAGES), reverse=True)
    return [[round(score, 3), page] for score, page in ranked[:int(how_many)]]


with gr.Blocks(title="Search by meaning") as demo:
    gr.Markdown("## Search by meaning\nAsk in English, ไทย or 中文.")
    with gr.Row():
        question = gr.Textbox(label="What are you looking for?", scale=3)
        how_many = gr.Slider(1, 9, value=3, step=1, label="How many", scale=1)
    results = gr.Dataframe(headers=["score", "page"], label="Closest pages",
                           wrap=True)

    # Searching costs milliseconds, so there is no need for a button.
    # change() runs as the person types.
    question.change(fn=find, inputs=[question, how_many], outputs=results)
    how_many.change(fn=find, inputs=[question, how_many], outputs=results)

    gr.Examples(examples=["how do I make a window", "ส่งอีเมลด้วย Python",
                          "让模型自己调用工具"],
                inputs=question)

if __name__ == "__main__":
    demo.launch()

find('how do I make a window', 3)
  [0.398, 'Classes in Python, taught by building a window piece by piece']
  [0.373, 'Build a chat window with tkinter and a model that remembers']
  [0.218, 'Turn a Python function into a web page with Gradio']

served at http://127.0.0.1:7895/   status 200, 17,768 bytes