🔎 Getting a GitHub Pages site into Google · พาเว็บ GitHub Pages เข้า Google · 让 GitHub Pages 网站进到 Google 里

1 The repository, and what Pages does not give you · รีโพ และสิ่งที่ Pages ไม่ได้ให้มา · 仓库，以及 Pages 没给你的东西

💻 MAKE IT · at the terminal, once

# The name matters. It must be exactly yourname.github.io
gh repo create yourname.github.io --public --clone
cd yourname.github.io

echo '<!doctype html><html lang="en"><head><title>Hello</title>'\
     '</head><body><h1>Hello</h1></body></html>' > index.html
git add index.html && git commit -m 'first page' && git push

# Settings -> Pages -> Source: main. Then wait a minute.

🖥️ WHAT A FRESH ONE LOOKS LIKE · measured on a real Pages site

https://octocat.github.io/            200
https://octocat.github.io/robots.txt  404   <- nothing there
https://octocat.github.io/sitemap.xml 404   <- nothing there

X-Robots-Tag: not set                       <- good, nothing blocks you

2 Read your own pages first · อ่านหน้าเว็บของตัวเองก่อน · 先读一遍你自己的页面

📄 NEW FILE · make_sample.py

"""Build a small site to test the scripts against, the shape a student's is.

Four pages, one of them deliberately missing its description and sharing a
title with another, so the checker in step 5 has something to find.
"""
import sys
from pathlib import Path

SITE = Path(__file__).parent / "site"
sys.stdout.reconfigure(encoding="utf-8")

PAGE = """<!doctype html>
<html lang="{lang}">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>{title}</title>
{description}
</head>
<body>
<h1>{heading}</h1>
<p>{body}</p>
</body>
</html>
"""

DESC = '<meta name="description" content="{0}">'

PAGES = {
    "index.html": dict(
        lang="en", title="Nari's notebook",
        description=DESC.format("Notes on learning Python in Chiang Mai, "
                                "with small programs you can run."),
        heading="Nari's notebook",
        # The front page links to one of the three. The other is an orphan,
        # which is what the last challenge goes looking for.
        body='Everything I have worked out so far. '
             '<a href="python.html">Learning Python</a>.'),
    "python.html": dict(
        lang="en", title="Learning Python",
        description=DESC.format("How I started with Python, and the first "
                                "five things that confused me."),
        heading="Learning Python", body="Start at the prompt, not in a file."),
    "khao-soi.html": dict(
        lang="en", title="Learning Python",
        description="",
        heading="Khao soi", body="The best bowl is on Nimmanhaemin."),
    "drafts/notes.html": dict(
        lang="en", title="Rough notes",
        description=DESC.format("Half-finished notes, not meant for anyone."),
        heading="Rough notes", body="Do not read this yet."),
}


def main():
    for name, fields in PAGES.items():
        target = SITE / name
        target.parent.mkdir(parents=True, exist_ok=True)
        target.write_text(PAGE.format(**fields), encoding="utf-8")
        print(f"  wrote {name}")
    print(f"{len(PAGES)} pages in {SITE}/")


if __name__ == "__main__":
    main()

📄 NEW FILE · check1.py

"""check1.py — read your own pages the way a search engine will."""
import html
import re
import sys
from collections import Counter
from pathlib import Path

FOLDER = Path("site")

# Google shows roughly this much. Longer is not an error; it is just cut off
# in the results, and the part it cuts is the part you wrote last.
TITLE_ROOM = 60
DESCRIPTION_ROOM = 155

sys.stdout.reconfigure(encoding="utf-8")


def first(pattern, text):
    """The first capture of a pattern, unescaped, or None."""
    found = re.search(pattern, text, re.I | re.S)
    return html.unescape(found.group(1)).strip() if found else None


def read(path):
    """Pull out the handful of things a search engine actually reads."""
    text = path.read_text(encoding="utf-8")
    return {
        "title": first(r"<title>(.*?)</title>", text),
        "description": first(r'<meta\s+name="description"\s+content="(.*?)"', text),
        "lang": first(r"<html[^>]*\blang=\"([^\"]+)\"", text),
        "h1": first(r"<h1[^>]*>(.*?)</h1>", text),
        "noindex": bool(re.search(r'name="robots"[^>]*noindex', text, re.I)),
    }


def main():
    pages = {path: read(path) for path in sorted(FOLDER.rglob("*.html"))}

    # Two pages with the same title look like one page to a search engine,
    # and it picks which to show. That choice is not usually yours.
    titles = Counter(page["title"] for page in pages.values() if page["title"])

    problems = 0
    for path, page in pages.items():
        found = []
        if not page["title"]:
            found.append("no title")
        elif titles[page["title"]] > 1:
            found.append(f"title shared with {titles[page['title']] - 1} other")
        elif len(page["title"]) > TITLE_ROOM:
            found.append(f"title is {len(page['title'])} characters, "
                         f"{TITLE_ROOM} get shown")

        if not page["description"]:
            found.append("no description, so Google writes one for you")
        elif len(page["description"]) > DESCRIPTION_ROOM:
            found.append(f"description is {len(page['description'])} characters")

        if not page["lang"]:
            found.append("no lang on <html>")
        if not page["h1"]:
            found.append("no <h1>")

        name = path.relative_to(FOLDER).as_posix()
        if found:
            problems = problems + len(found)
            print(f"{name}")
            for trouble in found:
                print(f"   - {trouble}")
        else:
            print(f"{name}  ok")

    print()
    print(f"{len(pages)} pages, {problems} things to fix")


if __name__ == "__main__":
    main()

🖥️ WHAT IT PRINTED · on the practice site

drafts/notes.html  ok
index.html  ok
khao-soi.html
   - title shared with 1 other
   - no description, so Google writes one for you
python.html
   - title shared with 1 other

4 pages, 3 things to fix

3 A sitemap, and a robots.txt that points at it · sitemap กับ robots.txt ที่ชี้ไปหามัน · 一份 sitemap，和一个指向它的 robots.txt

📄 NEW FILE · sitemap1.py

"""sitemap1.py — write the list of pages you want Google to know about."""
import sys
import time
from pathlib import Path

# Your site's address. The trailing slash matters; every URL is built on it.
SITE = "https://nari.github.io/"
FOLDER = Path("site")

# Folders whose pages should not be in the sitemap. A sitemap is a list of
# pages you are asking to have indexed, so drafts do not belong in it.
SKIP = {"drafts"}

sys.stdout.reconfigure(encoding="utf-8")


def pages():
    """Every .html file in the folder, as a web address, shortest first."""
    found = []
    for path in sorted(FOLDER.rglob("*.html")):
        parts = path.relative_to(FOLDER).parts
        if SKIP & set(parts[:-1]):
            continue

        # index.html is the folder itself. Listing both is one page twice.
        where = "/".join(parts[:-1] + ((), (parts[-1],))[parts[-1] != "index.html"])
        found.append((SITE + where, path))
    return found


def main():
    lines = ['<?xml version="1.0" encoding="UTF-8"?>',
             '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">']
    for url, path in pages():
        # The date the file changed. Google reads it as a hint, not a promise.
        when = time.strftime("%Y-%m-%d", time.gmtime(path.stat().st_mtime))
        lines.append(f"  <url><loc>{url}</loc><lastmod>{when}</lastmod></url>")
        print(f"  {url}")
    lines.append("</urlset>")

    target = FOLDER / "sitemap.xml"
    target.write_text("\n".join(lines) + "\n", encoding="utf-8")

    # robots.txt is how a search engine finds the sitemap without being told.
    robots = FOLDER / "robots.txt"
    robots.write_text("User-agent: *\nAllow: /\n\n"
                      f"Sitemap: {SITE}sitemap.xml\n", encoding="utf-8")

    print(f"{len(lines) - 3} pages -> {target}")
    print(f"and {robots}")


if __name__ == "__main__":
    main()

🖥️ WHAT IT PRINTED · three pages, one skipped

  https://nari.github.io/
  https://nari.github.io/khao-soi.html
  https://nari.github.io/python.html
3 pages -> site\sitemap.xml
and site\robots.txt

4 Check it as it is actually served · ตรวจมันในแบบที่ถูกเสิร์ฟจริง · 按它真正被送出的样子检查

📄 NEW FILE · live1.py

"""live1.py — check the site the way Google will, over the network.

Everything up to here read files on your disk. This reads the site as it
is actually served, which is the only version that counts.
"""
import re
import sys
import urllib.error
import urllib.request

SITE = sys.argv[1] if len(sys.argv) > 1 else "https://nari.github.io/"
if not SITE.endswith("/"):
    SITE = SITE + "/"

sys.stdout.reconfigure(encoding="utf-8")


def fetch(url):
    """Get a page, and the headers, without pretending to be a browser."""
    request = urllib.request.Request(url, headers={"User-Agent": "a lesson"})
    try:
        with urllib.request.urlopen(request, timeout=30) as response:
            return response.status, dict(response.headers), response.read()
    except urllib.error.HTTPError as refused:
        return refused.code, dict(refused.headers), b""
    except urllib.error.URLError as trouble:
        print("cannot reach", url, "-", trouble.reason)
        raise SystemExit(1)


def main():
    print("checking", SITE)

    status, headers, body = fetch(SITE)
    print(f"  the front page          {status}")

    # This header can hide a whole site from search, and nothing on the
    # page shows it. It is the first thing to rule out.
    blocked = headers.get("X-Robots-Tag", "")
    print(f"  X-Robots-Tag            {blocked or 'not set, good'}")

    status, _, robots = fetch(SITE + "robots.txt")
    print(f"  robots.txt              {status}")
    sitemap_line = ""
    if status == 200:
        text = robots.decode("utf-8", errors="replace")
        if re.search(r"^\s*Disallow:\s*/\s*$", text, re.M | re.I):
            print("     Disallow: / is in there. That hides everything.")
        found = re.search(r"^\s*Sitemap:\s*(\S+)", text, re.M | re.I)
        sitemap_line = found.group(1) if found else ""
        print(f"     names a sitemap      {sitemap_line or 'no'}")

    target = sitemap_line or SITE + "sitemap.xml"
    status, _, xml = fetch(target)
    print(f"  sitemap.xml             {status}")
    if status != 200:
        return

    urls = re.findall(r"<loc>\s*([^<\s]+)\s*</loc>",
                      xml.decode("utf-8", errors="replace"))
    print(f"     lists                {len(urls)} pages")

    # A sitemap full of dead links is worse than no sitemap. Check a few.
    bad = []
    for url in urls[:12]:
        code, _, _ = fetch(url)
        if code != 200:
            bad.append((code, url))
    print(f"     checked              {min(len(urls), 12)}, "
          f"{len(bad)} not reachable")
    for code, url in bad:
        print(f"       {code}  {url}")


if __name__ == "__main__":
    main()

🖥️ A SITE WITH ALL OF IT · run against a real one

checking https://krueng.ai/
  the front page          200
  X-Robots-Tag            not set, good
  robots.txt              200
     names a sitemap      https://krueng.ai/sitemap.xml
  sitemap.xml             200
     lists                113 pages
     checked              12, 0 not reachable

🖥️ AND A BARE GITHUB PAGES SITE · the same script

checking https://octocat.github.io/
  the front page          200
  X-Robots-Tag            not set, good
  robots.txt              404
  sitemap.xml             404

5 Proving the site is yours · พิสูจน์ว่าเว็บเป็นของคุณ · 证明这个站是你的

💻 THE FILE METHOD · the one that always works on Pages

# Search Console gives you a file named something like:
#   google1a2b3c4d5e6f7890.html
# Put it at the top of the repository, not in a folder.

git add google1a2b3c4d5e6f7890.html
git commit -m 'search console'
git push

# Wait for the build, then check it yourself before clicking Verify:
curl -I https://yourname.github.io/google1a2b3c4d5e6f7890.html

6 Handing over the sitemap, and waiting · ส่ง sitemap ให้ แล้วรอ · 把 sitemap 交上去，然后等

7 Try first. The answers are folded away. · ลองเองก่อน เฉลยพับซ่อนไว้ · 先自己试。答案折起来放着。

"""verify1.py — challenge 1: never submit a page that is not there.

sitemap1.py lists every file it finds. A file on your disk is not the same
as a page on your site: the name can differ by a capital letter, the push
can have failed, the build can have skipped it. Check first, list second.
"""
import sys
import time
import urllib.error
import urllib.request
from pathlib import Path

# Point it at your own site and the folder you publish from.
#   python verify1.py https://nari.github.io/ site
SITE = sys.argv[1] if len(sys.argv) > 1 else "https://nari.github.io/"
FOLDER = Path(sys.argv[2] if len(sys.argv) > 2 else "site")
SKIP = {"drafts"}

sys.stdout.reconfigure(encoding="utf-8")


def addresses():
    """The same list sitemap1.py would write."""
    found = []
    for path in sorted(FOLDER.rglob("*.html")):
        parts = path.relative_to(FOLDER).parts
        if SKIP & set(parts[:-1]):
            continue
        tail = "/".join(parts[:-1] + ((), (parts[-1],))[parts[-1] != "index.html"])
        found.append(SITE + tail)
    return found


def answers(url):
    """Does this address really return a page? Follow nothing, forgive nothing."""
    request = urllib.request.Request(url, method="HEAD",
                                     headers={"User-Agent": "a lesson"})
    try:
        with urllib.request.urlopen(request, timeout=20) as response:
            return response.status
    except urllib.error.HTTPError as refused:
        return refused.code
    except urllib.error.URLError:
        return 0


def main():
    good, bad = [], []
    for url in addresses():
        code = answers(url)
        (good if code == 200 else bad).append((code, url))
        print(f"  {code or 'dead':>4}  {url}")
        time.sleep(0.2)          # be polite to your own server

    print()
    print(f"{len(good)} would go in the sitemap, {len(bad)} would not")
    if bad:
        print("Fix these or drop them. A sitemap of 404s teaches Google")
        print("that your sitemap is not worth reading.")


if __name__ == "__main__":
    main()

python verify1.py https://krueng.ai/ ../HTML

496 would go in the sitemap, 3 would not

   404  https://krueng.ai/games/mueang-lanna
   404  https://krueng.ai/io2026
   404  https://krueng.ai/it/ai/plans

"""orphans1.py — challenge 2: find the pages nothing links to.

Google follows links. A page in your sitemap that no other page points at
is a page it will reach last and trust least, and usually it is a page you
forgot about rather than one you meant to hide.
"""
import re
import sys
from pathlib import Path

FOLDER = Path("site")
sys.stdout.reconfigure(encoding="utf-8")


def linked_from(path):
    """Every local page this one points at, as a path inside the folder."""
    text = path.read_text(encoding="utf-8")
    here = path.parent
    out = set()
    for href in re.findall(r'<a[^>]+href="([^"]+)"', text, re.I):
        # Leave the outside world, anchors and mail links alone.
        if href.startswith(("http://", "https://", "#", "mailto:", "//")):
            continue
        target = (here / href.split("#")[0].split("?")[0]).resolve()
        out.add(target)
    return out


def main():
    pages = sorted(FOLDER.rglob("*.html"))
    pointed_at = set()
    for path in pages:
        pointed_at |= linked_from(path)

    front = (FOLDER / "index.html").resolve()
    orphans = [p for p in pages
               if p.resolve() not in pointed_at and p.resolve() != front]

    print(f"{len(pages)} pages, {len(pointed_at)} link targets found")
    print()
    if not orphans:
        print("nothing is orphaned")
        return
    print("nothing links to these:")
    for path in orphans:
        print("   ", path.relative_to(FOLDER).as_posix())
    print()
    print("Either link to them from a page that is already indexed,")
    print("or take them out of the sitemap. Do not do neither.")


if __name__ == "__main__":
    main()

4 pages, 1 link targets found

nothing links to these:
    drafts/notes.html
    khao-soi.html

Either link to them from a page that is already indexed,
or take them out of the sitemap. Do not do neither.

# .github/workflows/sitemap.yml
# Challenge 3: rebuild the sitemap every time you push, so it is never
# one page behind the site.
name: Rebuild the sitemap

on:
  push:
    branches: [main]

# A workflow can read the repository by default. Writing back to it needs
# saying so here, and this one commits the sitemap it just built.
permissions:
  contents: write

jobs:
  sitemap:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v5
        with:
          python-version: "3.12"

      - name: Write sitemap.xml and robots.txt
        run: python sitemap1.py

      - name: Commit them, but only if they changed
        run: |
          git config user.name "github-actions[bot]"
          git config user.email "github-actions[bot]@users.noreply.github.com"
          git add site/sitemap.xml site/robots.txt
          # --quiet returns non-zero when there is something staged, so this
          # commits only when there is a real change. Without it the job
          # fails on every push that did not touch a page.
          git diff --staged --quiet || git commit -m "sitemap: rebuild"
          git push

"""tell1.py — tell a search engine a page changed, without waiting.

Google used to take a ping at /ping?sitemap=. That endpoint now answers
404; they removed it and say they read sitemaps on their own schedule.
Bing and Yandex kept a way in, called IndexNow, and it is two steps: put
a key file on your site, then name the pages that changed.
"""
import json
import secrets
import sys
import urllib.error
import urllib.request
from pathlib import Path

SITE = "https://nari.github.io/"
FOLDER = Path("site")
INDEXNOW = "https://api.indexnow.org/indexnow"

sys.stdout.reconfigure(encoding="utf-8")


def key_file():
    """Make the key once and keep it. It lives in the site, so it is public."""
    saved = FOLDER / "indexnow-key.txt"
    if saved.exists():
        return saved.read_text(encoding="utf-8").strip()

    # Any 8 to 128 hex characters. It proves you can write to the site,
    # which is the only thing it needs to prove.
    key = secrets.token_hex(16)
    saved.write_text(key, encoding="utf-8")
    (FOLDER / (key + ".txt")).write_text(key, encoding="utf-8")
    print("made a new key and wrote", key + ".txt")
    return key


def live(url):
    """Is this address actually reachable? Ask before claiming anything."""
    try:
        with urllib.request.urlopen(url, timeout=20) as response:
            return response.status == 200
    except (urllib.error.URLError, urllib.error.HTTPError):
        return False


def main():
    key = key_file()
    proof = SITE + key + ".txt"
    print("key file should be at", proof)

    if not live(proof):
        # Sending before the key file is published gets the whole request
        # rejected, and the reason is not obvious from the error.
        print("that address does not answer yet.")
        print("Commit the key file, push, wait for Pages to build, then rerun.")
        return

    changed = [SITE, SITE + "python.html"]
    body = json.dumps({"host": SITE.split("/")[2], "key": key,
                       "urlList": changed}).encode()
    request = urllib.request.Request(
        INDEXNOW, data=body, headers={"Content-Type": "application/json"})
    try:
        with urllib.request.urlopen(request, timeout=30) as response:
            print("told IndexNow about", len(changed), "pages:", response.status)
    except urllib.error.HTTPError as refused:
        # 422 means it found the key file but did not like something.
        print("refused:", refused.code, refused.read().decode()[:200])


if __name__ == "__main__":
    main()

made a new key and wrote b44cba0f1f6002f53edc1da1eea9a2e1.txt
key file should be at
  https://nari.github.io/b44cba0f1f6002f53edc1da1eea9a2e1.txt
that address does not answer yet.
Commit the key file, push, wait for Pages to build, then rerun.