"""check_questions.py - check questions.json against the lesson pages. python check_questions.py Run it from this folder after you add or change a question. It starts no browser and needs no library. It ends with code 1 when something is wrong. It checks that every topic has enough questions for a round, that every question has one right answer, that a typed answer is one word, that no question names its lesson ("the lesson", "step 6", "Mission A": a question tests the technology and stands alone), that each link goes to a page and an anchor that exist, and it counts how often the right answer is the longest option. Then it checks questions_zh.json: the same questions, the same right answers and the same code as questions.json, in Chinese. """ import collections import json import pathlib import re import sys PER_ROUND = 5 # questions that one topic gets in a round LONGEST_LIMIT = 0.40 # the right answer may be the longest in this share of questions def anchors_of(page): return set(re.findall(r'\bid="([^"]+)"', page.read_text(encoding="utf-8"))) def check_link(html, link, anchors, problems, where): path, _, anchor = link.partition("#") page = html / path if not page.exists(): problems.append(f"{where}: there is no page {path}") return if path not in anchors: anchors[path] = anchors_of(page) if anchor and anchor not in anchors[path]: problems.append(f"{where}: {path} has no id=\"{anchor}\"") # A question tests the technology and stands alone: it does not name its lesson # (owner's rule). These words in a question, an option or an explanation show # that it still leans on one. LESSON_WORDS = re.compile( r"\b(the|this|that) (lesson|page|tutorial)\b|\blesson'?s\b|\bcheat sheet\b|" r"\b(Mission|Tutorial|Challenge) [A-Z0-9]\b|\b[Ss]tep \d|\b(we|our)\b") def code_in(text): """The pieces between backticks: code, which is the same in every language.""" return sorted(re.findall(r"`([^`]+)`", text)) def check_chinese(bank, chinese, problems): """The Chinese file must be the English file with other words, and nothing else.""" if [u["id"] for u in bank["units"]] != [u["id"] for u in chinese["units"]]: problems.append("questions_zh.json: the units are not those of questions.json") if len(bank["questions"]) != len(chinese["questions"]): problems.append("questions_zh.json: not the same number of questions") same = 0 for english, other in zip(bank["questions"], chinese["questions"]): name = english["id"] for field in ("id", "unit", "d", "type", "a", "answers", "code", "at"): if english.get(field) != other.get(field): problems.append(f"{name}: questions_zh.json has another {field}") if len(english.get("o", [])) != len(other.get("o", [])): problems.append(f"{name}: questions_zh.json has another number of options") if len(set(other.get("o", []))) != len(other.get("o", [])): problems.append(f"{name}: two options are the same in Chinese") pieces = [english["q"], english["why"]] + english.get("o", []) others = [other["q"], other["why"]] + other.get("o", []) for one, two in zip(pieces, others): if code_in(one) != code_in(two): problems.append(f"{name}: the code in the Chinese is not the code in the English: " f"{code_in(one)} / {code_in(two)}") if not re.search(r"[一-鿿]", other["q"] + other["why"]): problems.append(f"{name}: no Chinese in questions_zh.json") if other["q"] == english["q"]: same += 1 print(len(chinese["questions"]), "questions in Chinese.", same, "still have the English question.") def main(): here = pathlib.Path(__file__).resolve().parent html = here.parent.parent bank = json.loads((here.parent / "questions.json").read_text(encoding="utf-8")) problems = [] anchors = {} units = {} for unit in bank["units"]: for field in ("id", "group", "label", "at"): if not unit.get(field): problems.append(f"unit {unit.get('id')}: no {field}") units[unit["id"]] = 0 check_link(html, unit["at"], anchors, problems, "unit " + unit["id"]) seen = set() stems = set() longest = 0 choices = 0 places = collections.Counter() for question in bank["questions"]: name = question.get("id", "?") if name in seen: problems.append(f"{name}: the id is used twice") seen.add(name) # two lessons may share a question, such as the Mac and the Windows page if (question.get("unit"), question["q"]) in stems: problems.append(f"{name}: the same question is asked twice in one topic") stems.add((question.get("unit"), question["q"])) if question.get("unit") not in units: problems.append(f"{name}: unknown unit {question.get('unit')}") continue units[question["unit"]] += 1 if not -1 <= question.get("d", 9) <= 1: problems.append(f"{name}: the difficulty must be from -1 to 1") if not question.get("why"): problems.append(f"{name}: no explanation") words = " ".join([question["q"], question.get("why", "")] + question.get("o", [])) found = LESSON_WORDS.search(words) if found: problems.append(f"{name}: it refers to its lesson: \"{found.group(0)}\"") check_link(html, question["at"], anchors, problems, name) if question["type"] == "fill": answers = question.get("answers") or [] if not answers or any(one != one.lower().strip() for one in answers): problems.append(f"{name}: answers must be lowercase, and at least one") # A typed answer is one word (owner's rule). The first answer is the one # the page shows; a longer spelling may follow it, to be kind to the student. # A question that needs more than one word is a multiple-choice question. elif " " in answers[0]: problems.append(f"{name}: the typed answer \"{answers[0]}\" is more than one word") elif question["type"] == "mcq": options = question.get("o") or [] if len(options) != 4 or len(set(options)) != 4: problems.append(f"{name}: needs four different options") continue if question.get("a") not in (0, 1, 2, 3): problems.append(f"{name}: a must be 0, 1, 2 or 3") continue lengths = [len(one) for one in options] right = lengths[question["a"]] choices += 1 places[question["a"]] += 1 if right == max(lengths) and lengths.count(right) == 1: longest += 1 # short options, such as four command names, give nothing away if right > 25 and right > 1.6 * sorted(lengths)[-2]: problems.append(f"{name}: the right answer is much longer than the others") else: problems.append(f"{name}: unknown type {question['type']}") for unit, count in units.items(): if count < PER_ROUND: problems.append(f"unit {unit}: {count} questions, and a round needs {PER_ROUND}") print(len(bank["questions"]), "questions on", len(units), "topics.") print("The right answer is at place 1, 2, 3, 4 in:", [places[i] for i in range(4)]) share = longest / choices if choices else 0 print(f"The right answer is the longest option in {longest} of {choices} ({share:.0%}).", "By chance it would be 25%.") if share > LONGEST_LIMIT: problems.append(f"the right answer is the longest too often: over {LONGEST_LIMIT:.0%}") chinese = here.parent / "questions_zh.json" if chinese.exists(): check_chinese(bank, json.loads(chinese.read_text(encoding="utf-8")), problems) for problem in problems: print("PROBLEM:", problem) print(len(problems), "problem(s).") return 1 if problems else 0 if __name__ == "__main__": sys.exit(main())