Skills spend fewer rows and stop at a refusal, and the scripts move to TypeScript

.github/workflows/check.yml

@@ -22,7 +22,9 @@ jobs:
           GH_TOKEN: ${{ github.token }}
           COMMITS: ${{ github.event.pull_request.commits_url }}
         run: "! gh api --paginate \"$COMMITS\" --jq '.[].commit.message' | grep -iE '^(co-authored-by: .*(claude|anthropic)|claude-session:)'"
-      - run: node scripts/check-skills.mjs
-      - run: node scripts/check-contract.mjs
+      - run: npm ci
+      - run: npx tsc
+      - run: node scripts/check-skills.ts
+      - run: node scripts/check-contract.ts
       - run: npm install -g @anthropic-ai/claude-code @openai/codex
       - run: scripts/test-installs.sh

.gitignore

@@ -0,0 +1 @@
+node_modules/

AGENTS.md

@@ -1,3 +1,3 @@
 # Agents
 
-Read [CONTRIBUTING.md](CONTRIBUTING.md) before changing a skill. Run `node scripts/check-skills.mjs` when you finish.
+Read [CONTRIBUTING.md](CONTRIBUTING.md) before changing a skill. Run `node scripts/check-skills.ts` when you finish.

CONTRIBUTING.md

@@ -11,18 +11,19 @@ skills/<name>/
   LICENSE.txt
 shared/           data.md, cli.md, mcp.md and api.md, copied into every skill's references/
 evals/            Routing queries and traps
-scripts/          check-skills.mjs, check-contract.mjs and test-installs.sh
+scripts/          check-skills.ts, check-contract.ts and test-installs.sh
+package.json      TypeScript, for type-checking the scripts. Node runs them directly
 .claude-plugin/   Claude Code plugin and marketplace
 .codex-plugin/    Codex plugin
 .agents/plugins/  Codex marketplace
 ```
 
 ## Rules
 
-`scripts/check-skills.mjs` enforces the first three and the last.
+`scripts/check-skills.ts` enforces the first three and the last.
 
 - A skill works alone. A single-skill install copies only that skill's folder, so a skill can't link to anything outside it.
-- Skills name operations, such as `listings_versions`. Commands, tools and paths live in `shared/`. Edit `shared/`, then run `node scripts/check-skills.mjs --write` to copy it into every skill.
+- Skills name operations, such as `listings_versions`. Commands, tools and paths live in `shared/`. Edit `shared/`, then run `node scripts/check-skills.ts --write` to copy it into every skill.
 - Each store file starts with `Checked: YYYY-MM-DD against <url>`.
 - `SKILL.md` holds the steps. Anything only some runs need goes in `references/`, with a step in `SKILL.md` saying when to read it.
 - A report dates every number it reads, gives its basis, and says what it couldn't read. Leave out a claim you can't back.
@@ -31,8 +32,9 @@ scripts/          check-skills.mjs, check-contract.mjs and test-installs.sh
 ## Before you open a pull request
 
 ```bash
-node scripts/check-skills.mjs
-node scripts/check-contract.mjs
+npm ci && npx tsc
+node scripts/check-skills.ts
+node scripts/check-contract.ts
 ```
 
 ```bash
@@ -41,6 +43,6 @@ scripts/test-installs.sh
 
 The second installs the skills into clean folders, as plugins for Claude Code and Codex and through `npx skills` for 11 agents, and checks that each one got all of them. CI runs both. Add `--run` to also ask each signed-in agent on your machine to use a skill, which spends model credits.
 
-If you change a skill's description, run its queries from `evals/confusion-pool.json` with `evals/run_confusion.py`. If you change what a skill does, run its row in `evals/traps.md`.
+If you change a skill's description, run its queries from `evals/confusion-pool.json` with `evals/run-confusion.ts`. If you change what a skill does, run its row in `evals/traps.md`.
 
 Commits and pull requests carry no AI attribution. `.claude/settings.json` turns it off in Claude Code, and CI fails a pull request whose commits carry it.

evals/run-confusion.ts

@@ -0,0 +1,79 @@
+#!/usr/bin/env node
+// Runs a labelled query pool with all eight skills installed, and records the first tool each run calls.
+//   node evals/run-confusion.ts <pilot dir> <pool.json> <out.json> <model> <runs> [max turns]
+import { spawn } from "node:child_process";
+import { readFileSync, writeFileSync } from "node:fs";
+
+type Query = { query: string; expect: string };
+type Run = { first: string; skills: unknown };
+
+const [pilot, poolPath, out, model, runsArg, turns = "1"] = process.argv.slice(2);
+if (!pilot || !poolPath || !out || !model || !runsArg) throw new Error("usage: run-confusion.ts <pilot dir> <pool.json> <out.json> <model> <runs> [max turns]");
+const pool: Query[] = JSON.parse(readFileSync(poolPath, "utf8"));
+const runs = Number(runsArg);
+const SKILLS = new Set(["plugin-stack", "optimize-listing", "submit-plugin", "audit-discoverability", "design-widgets", "watch-competitors", "find-opening", "track-launch"]);
+
+const claude = (query: string) => new Promise<string>((resolve, reject) => {
+  const child = spawn("claude", ["-p", query, "--output-format", "stream-json", "--verbose", "--max-turns", turns, "--model", model,
+    "--setting-sources", "project", "--strict-mcp-config", "--mcp-config", '{"mcpServers":{}}'], { cwd: pilot, stdio: ["ignore", "pipe", "ignore"], timeout: 170_000 });
+  let stdout = "";
+  child.stdout.on("data", (chunk) => { stdout += chunk; });
+  child.on("error", reject);
+  child.on("close", (_code, signal) => signal ? reject(new Error(`claude stopped by ${signal}`)) : resolve(stdout));
+});
+
+const firstTool = (use: { name: string; input: Record<string, unknown> }) => {
+  const path = String(use.input.file_path ?? "");
+  if (use.name === "Skill") return String(use.input.skill);
+  if (use.name === "Read" && path.includes(".claude/skills/")) return path.split(".claude/skills/")[1]!.split("/")[0]!;
+  return `tool:${use.name}`;
+};
+
+const run = async (query: string): Promise<Run> => {
+  let first: string | undefined, skills: unknown;
+  for (const line of (await claude(query)).split("\n")) {
+    let event: any;
+    try { event = JSON.parse(line); } catch { continue; }
+    if (event.type === "system" && event.subtype === "init") skills = event.skills;
+    if (event.type === "assistant" && (first === undefined || first.startsWith("tool:"))) {
+      const use = event.message.content.find((c: { type: string }) => c.type === "tool_use");
+      if (use) first = firstTool(use);
+    }
+  }
+  return { first: first ?? "text", skills };
+};
+
+// Six at once, each answer printed in the pool's order.
+const answers: Promise<Run[]>[] = [];
+const waiting: (() => void)[] = [];
+let active = 0;
+const job = async (item: Query) => {
+  if (active >= 6) await new Promise<void>((go) => waiting.push(go));
+  active++;
+  try {
+    const rs: Run[] = [];
+    for (let i = 0; i < runs; i++) rs.push(await run(item.query));
+    return rs;
+  } finally { active--; waiting.shift()?.(); }
+};
+for (const item of pool) answers.push(job(item));
+
+const results: (Query & { got: string[] })[] = [];
+let installed: unknown;
+for (const [i, item] of pool.entries()) {
+  const rs = await answers[i]!, got = rs.map((r) => r.first);
+  results.push({ ...item, got });
+  installed = rs[0]?.skills;
+  console.log(item.expect.slice(0, 10).padEnd(11), got, item.query.slice(0, 70));
+}
+writeFileSync(out, JSON.stringify({ model, installed, results }, null, 1));
+
+const hit: Record<string, number> = {}, total: Record<string, number> = {}, confused: Record<string, number> = {};
+for (const r of results) for (const g of r.got) {
+  const expected = r.expect.split("|"), ok = expected.includes(g) || (expected.includes("none") && !SKILLS.has(g));
+  total[r.expect] = (total[r.expect] ?? 0) + 1;
+  hit[r.expect] = (hit[r.expect] ?? 0) + (ok ? 1 : 0);
+  if (!ok) confused[`${r.expect} -> ${g}`] = (confused[`${r.expect} -> ${g}`] ?? 0) + 1;
+}
+console.log(Object.fromEntries(Object.keys(total).map((k) => [k, `${hit[k]}/${total[k]}`])));
+console.log(Object.entries(confused).sort((a, b) => b[1] - a[1]));