diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..409a406 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,18 @@ +root = true + +[*] +charset = utf-8 +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true + +[*.py] +indent_style = space +indent_size = 4 + +[*.{yml,yaml,json,astro,md,mjs}] +indent_style = space +indent_size = 2 + +[*.md] +trim_trailing_whitespace = false diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..e61085f --- /dev/null +++ b/.gitattributes @@ -0,0 +1,3 @@ +# Every tracked text file is stored with LF; keep working trees the same on +# Windows and macOS. +* text=auto eol=lf diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 65464a5..c90df99 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,6 +1,4 @@ # Default owners for everything in the repo. -# TODO: replace with the maintainer's GitHub handle or org team before Phase 1. * @GetTechAPI/maintainers -/data/ @GetTechAPI/maintainers /app/ @GetTechAPI/maintainers diff --git a/.github/workflows/final-check.yml b/.github/workflows/final-check.yml index 7359654..f8f6be9 100644 --- a/.github/workflows/final-check.yml +++ b/.github/workflows/final-check.yml @@ -4,7 +4,7 @@ name: final-check # runs what test.yml runs (ruff, mypy, the seed-data validator, pytest with # coverage) on the PR, and TechEngineBot leaves ONE comment with the verdict; # adding the label again edits that comment. This is the only place the -# TechEngineBot account is used in this repo, and it replaces Copilot reviews. +# TechEngineBot account is used by a workflow in this repo. # # The check job runs PR code, so it gets no secrets and a read-only token. # Only the comment job sees TECHENGINEBOT_TOKEN, which must be a fine-grained diff --git a/.github/workflows/ingest.yml b/.github/workflows/ingest.yml index bb68923..72f068e 100644 --- a/.github/workflows/ingest.yml +++ b/.github/workflows/ingest.yml @@ -188,8 +188,9 @@ jobs: PY # The token is read-only, so the commit leaves the run as a patch. A - # maintainer opens the PR from their own account with TechAPI's - # .github/scripts/open-collect-pr.sh , so the PR's checks run. + # maintainer runs TechAPI's .github/scripts/open-collect-pr.sh : + # their account pushes the branch and TechEngineBot opens the PR, so the + # PR's checks run. - name: Export the commit as collect.patch if: steps.commit.outputs.changed == 'true' env: diff --git a/.github/workflows/refresh.yml b/.github/workflows/refresh.yml index 0c1ee55..307f865 100644 --- a/.github/workflows/refresh.yml +++ b/.github/workflows/refresh.yml @@ -206,8 +206,9 @@ jobs: PY # The token is read-only, so the commit leaves the run as a patch. A - # maintainer opens the PR from their own account with TechAPI's - # .github/scripts/open-collect-pr.sh , so the PR's checks run. + # maintainer runs TechAPI's .github/scripts/open-collect-pr.sh : + # their account pushes the branch and TechEngineBot opens the PR, so the + # PR's checks run. - name: Export the commit as collect.patch if: steps.commit.outputs.changed == 'true' env: diff --git a/.github/workflows/verify-promote.yml b/.github/workflows/verify-promote.yml index 168582e..4ec8d25 100644 --- a/.github/workflows/verify-promote.yml +++ b/.github/workflows/verify-promote.yml @@ -221,8 +221,9 @@ jobs: PY # The token is read-only, so the commit leaves the run as a patch. A - # maintainer opens the PR from their own account with TechAPI's - # .github/scripts/open-collect-pr.sh , so the PR's checks run. + # maintainer runs TechAPI's .github/scripts/open-collect-pr.sh : + # their account pushes the branch and TechEngineBot opens the PR, so the + # PR's checks run. - name: Export the commit as collect.patch if: steps.commit.outputs.changed == 'true' env: diff --git a/.gitignore b/.gitignore index 2ab09b1..1c9813c 100644 --- a/.gitignore +++ b/.gitignore @@ -19,15 +19,16 @@ env/ *.sqlite *.sqlite3 +# A TechAPI checkout placed inside this repo (CI does this and sets +# TECHAPI_DATA_DIR); the dataset is never committed here. +/TechAPI/ + # Generated static dump (build artifact; published by refresh-data workflow) /dump/ # Local-only handoff notes (private; never committed to the public repo) *.local.md -# Local tooling/editor config (kept out of the public repo) -.claude/ - # Note: data/_staging/ (raw collected candidate pool) is intentionally tracked — # comprehensive data collection is a purpose of this repo. diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index a68a955..0000000 --- a/.gitmodules +++ /dev/null @@ -1,4 +0,0 @@ -[submodule "TechAPI"] - path = TechAPI - url = https://github.com/GetTechAPI/TechAPI.git - branch = main diff --git a/README.md b/README.md index 26664a8..d369aff 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,9 @@ TechEngine owns everything *around* the data: schema validation, the FastAPI read API, the static JSON dump generator, the engine's own landing site, -automated coverage checks, and the weekly ingestion/refresh crawlers. +coverage checks, the ingestion/refresh collectors and the verification tiers. +Nothing here runs on a schedule; collectors run only when a maintainer +dispatches them. The dataset and the public-facing playground site live in [TechAPI](https://github.com/GetTechAPI/TechAPI) so each can be versioned, @@ -17,52 +19,81 @@ engine's own landing: what TechEngine is, what it runs, and links out to docs. ## Layout ```text -app/ - validate.py # schema/range/uniqueness checks - seed.py # data/ -> SQLModel database - dump.py # API -> static JSON tree - main.py # FastAPI entrypoint +app/ # the engine (the one installable Python package) + main.py # FastAPI entry point; mounts every router under /v1 + routers/ # /v1 routes, one module per collection (list below) models/ # SQLModel tables - routers/ # /v1/{brands,socs,smartphones,gpus,cpus} - schemas/ # Pydantic response models - services/ # scoring (algorithm_version-tagged) - coverage/ # upstream-vs-curated diff + Markdown report - ingest/ # draft new records from upstream pages -tests/ # unit + integration -site/ # Astro engine landing (deploys to Pages) -docs/ # SPEC / DATA_PIPELINE / DEVELOPMENT -TechAPI/ # submodule -> GetTechAPI/TechAPI (clickable @ link) -.github/workflows/ - test.yml # lint + type-check + tests - validate-data.yml # read-only data validation (workflow_call or manual) - deploy-pages.yml # build & deploy engine site + dump - coverage-report.yml # manual: gap report to the job summary + artifact - ingest.yml # workflow_call: draft new SKUs -> collect.patch - refresh.yml # workflow_call: re-scrape benchmarks -> collect.patch - verify-promote.yml # workflow_call: Tier 1-3 checks + verified promotion patch - final-check.yml # label-triggered checks + one TechEngineBot comment + schemas/ # Pydantic response models + serializers + services/scoring/ # 0-100 scores, weights from config/scoring.yaml + data_root.py # finds the TechAPI dataset (see below) + categories.py # the 13 dataset categories and their collection names + validate.py # schema/range/uniqueness checks on the dataset + seed.py # dataset -> SQL database + dump.py # API -> static JSON tree (./dump) + coverage/ # upstream-vs-curated gap report + ingest/ # collectors: draft new records, refresh benchmarks + verify/ # trust tiers 0-3, verified promotion, backfill tools +config/scoring.yaml # scoring weights and reference scales +integrity_check.py # whole-dataset anomaly scan (the --strict gate) +tests/ # unit/, integration/, verify/ +site/ # Astro landing page, deployed to GitHub Pages +docs/ # SPEC, DATA_PIPELINE, DEVELOPMENT +.github/workflows/ # CI; every workflow is listed below +Dockerfile, docker-compose.yml # stale, do not build (see Docker) ``` -## How The Two Repos Connect +Routes under `/v1`: `health`, `version`, `brands`, `socs`, `smartphones`, +`tablets`, `watches`, `pdas`, `gpus`, `cpus`, `laptops`, `monitors`, +`software`, `websites`, `device-catalog`. Dataset folders are singular +(`cpu/`), routes are plural (`/v1/cpus`); `app/categories.py` holds the mapping. + +**How the engine finds the dataset.** This repo holds no data. Every entry +point asks `app/data_root.py`, which returns the first of: the +`TECHAPI_DATA_DIR` environment variable; `../data` (this repo checked out as +TechAPI's `TechEngine` submodule); `../TechAPI/data` (the two repos cloned side +by side). CI checks TechAPI out into `./TechAPI`, which is git-ignored, and +sets `TECHAPI_DATA_DIR` to its `data/` folder. + +Workflows: + +```text +test.yml # lint + type-check + tests +validate-data.yml # read-only data validation (workflow_call or manual) +deploy-pages.yml # on push to main: build site/ + static dump -> Pages +coverage-report.yml # manual: gap report to the job summary + artifact +ingest.yml # workflow_call: draft new SKUs -> collect.patch +refresh.yml # workflow_call: re-scrape benchmarks -> collect.patch +verify-promote.yml # workflow_call: Tier 1-3 checks + verified promotion patch +final-check.yml # label-triggered checks + one TechEngineBot comment +``` -Both repos live in the **GetTechAPI** org and each includes the other as a git -**submodule** (a clickable `@ ` pin). Nothing syncs them automatically. +## How The Two Repos Connect -No workflow here runs on a schedule or uses a personal access token (ADR-017). -The one comment any workflow posts is TechEngineBot's final check, and only when -a maintainer adds the `final-check` label to a PR (`final-check.yml`). +Both repos live in the **GetTechAPI** org. TechAPI pins this repo as its +`TechEngine` git **submodule** (a clickable `@ ` pin); this repo does not +pin TechAPI back, so there is no submodule loop. Nothing syncs them +automatically: the pin moves only when a TechAPI commit moves it. + +No workflow here runs on a schedule or uses a personal access token (ADR-017), +and no workflow opens or approves a PR. TechEngineBot opens PRs by hand from a +maintainer's machine, with its gh login kept apart from the maintainer's own; +the maintainer's account still commits, pushes and merges. The one comment any +workflow posts is TechEngineBot's final check, and only when a maintainer adds +the `final-check` label to a PR (`final-check.yml`). Collection runs only when a maintainer dispatches TechAPI's `collect.yml`, which calls `ingest.yml`, `refresh.yml` or `verify-promote.yml` with a read-only token. Each run commits locally, gates the result with `app.validate` and `integrity_check.py --strict`, and uploads the commit as `collect.patch`. A -maintainer opens the PR from their own account with TechAPI's -`.github/scripts/open-collect-pr.sh`; no workflow pushes or opens a PR. The +maintainer then runs TechAPI's `.github/scripts/open-collect-pr.sh` by hand: +their account pushes the run's `collect/*` branch and TechEngineBot opens the +PR. No workflow pushes or opens a PR. The engine that runs is the TechEngine commit TechAPI pins as its submodule. Crawlers identify themselves with a `TechEngine-*` User-Agent. Every Python entry point reads data from a **TechAPI checkout**. The location can be overridden via `TECHAPI_DATA_DIR`; by default TechEngine supports both -layouts: `TechAPI/TechEngine` as a submodule and `TechAPI` beside `TechEngine`. +layouts: `TechAPI/TechEngine` as TechAPI's submodule and `TechAPI` beside +`TechEngine` (see Layout). ## Quickstart @@ -70,7 +101,7 @@ layouts: `TechAPI/TechEngine` as a submodule and `TechAPI` beside `TechEngine`. git clone https://github.com/GetTechAPI/TechAPI.git ../TechAPI # data source pip install -e ".[dev]" python -m app.validate # check data integrity -python -m app.seed # data/ -> ./techapi.db (SQLite) +python -m app.seed # dataset -> ./techapi.db (SQLite) uvicorn app.main:app --reload # serve; curl localhost:8000/v1/cpus/ryzen-9-9950x3d python -m app.dump # generate ./dump/v1/... static tree ``` @@ -82,13 +113,13 @@ seed records may omit them or use `null` when no source value is known, and API detail responses and static dumps preserve unknown values as `null`. Smartphones continue to require numeric values for both fields. -### Docker Compose (Postgres) - -```bash -docker compose up --build -``` +### Docker -Spins up Postgres 16, seeds from the mounted TechAPI checkout, serves on `:8000`. +`Dockerfile` and `docker-compose.yml` predate the split from TechAPI and do +not work as they are: the Dockerfile copies `data/` and `scripts/`, which no +longer exist here, so the image does not build, and the compose file mounts no +TechAPI checkout for `app.seed` to read. No workflow builds or uses them. Use +the Quickstart above; for Postgres, point `DATABASE_URL` at your own instance. ## Roadmap diff --git a/TechAPI b/TechAPI deleted file mode 160000 index 0d083df..0000000 --- a/TechAPI +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 0d083df03d8971ce379aca7d42b66fd5a76b66a3 diff --git a/app/data_root.py b/app/data_root.py index 1eadd85..955c77a 100644 --- a/app/data_root.py +++ b/app/data_root.py @@ -1,4 +1,9 @@ -"""Resolve the TechAPI data directory for sibling and submodule layouts.""" +"""Resolve the TechAPI data directory. + +Order: ``TECHAPI_DATA_DIR``; then ``../data`` (this repo checked out as +TechAPI's ``TechEngine`` submodule); then ``../TechAPI/data`` (the two repos +cloned side by side). +""" from __future__ import annotations diff --git a/app/dump_check.py b/app/dump_check.py deleted file mode 100644 index 547ec2b..0000000 --- a/app/dump_check.py +++ /dev/null @@ -1,85 +0,0 @@ -"""Lightweight validation of a changed static dump against seed record counts.""" - -from __future__ import annotations - -import argparse -import json -import subprocess -from pathlib import Path -from typing import Any - -from app.categories import CATEGORIES, COLLECTIONS - - -def check_dump(repo: Path, base: str) -> list[str]: - changed = subprocess.run( - ["git", "diff", "--name-only", "--diff-filter=ACMR", f"{base}...HEAD", - "--", "site/public/v1/"], - cwd=repo, text=True, capture_output=True, check=True, - ).stdout.splitlines() - touched = subprocess.run( - ["git", "diff", "--name-only", f"{base}...HEAD", "--", "site/public/v1/"], - cwd=repo, text=True, capture_output=True, check=True, - ).stdout.splitlines() - if not touched: - return [] - errors: list[str] = [] - - def read(path: Path) -> Any: - try: - return json.loads(path.read_text(encoding="utf-8-sig")) - except (OSError, ValueError) as exc: - errors.append(f"{path.relative_to(repo).as_posix()}: {exc}") - return None - - for rel in changed: - if rel.endswith(".json"): - read(repo / rel) - root = repo / "site/public/v1" - manifest = read(root / "index.json") - collections = manifest.get("collections", {}) if isinstance(manifest, dict) else {} - if not isinstance(collections, dict): - errors.append("manifest collections must be an object") - collections = {} - for category in CATEGORIES: - resource = COLLECTIONS[category] - count = sum(1 for p in (repo / "data" / category).rglob("*.json") - if not p.name.startswith("_")) - entry = collections.get(resource) - index = read(root / resource / "index.json") - if not isinstance(entry, dict) or entry.get("count") != count: - errors.append(f"{resource}: manifest count must equal {count}") - if (not isinstance(index, dict) or index.get("count") != count - or not isinstance(index.get("results"), list) - or len(index["results"]) != count): - errors.append(f"{resource}: index count/results must equal {count}") - return errors - - -def scope(repo: Path, base_ref: str, base_sha: str) -> str: - count = sum(1 for category in CATEGORIES - for p in (repo / "data" / category).rglob("*.json") - if not p.name.startswith("_")) - n = len(CATEGORIES) - return f"{n}/{n} categories ? {count:,} records ? diff base {base_ref}@{base_sha[:7]}" - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--repo", type=Path, default=Path("TechAPI")) - parser.add_argument("--base", required=True) - parser.add_argument("--base-ref", default="unknown") - parser.add_argument("--scope-only", action="store_true") - args = parser.parse_args() - print(scope(args.repo, args.base_ref, args.base)) - if args.scope_only: - return 0 - errors = check_dump(args.repo, args.base) - for error in errors: - print(error) - print(f"Dump JSON/count check: {'FAIL' if errors else 'PASS (or no dump changes)'}") - return int(bool(errors)) - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/app/ingest/__init__.py b/app/ingest/__init__.py index 16b884a..c92be48 100644 --- a/app/ingest/__init__.py +++ b/app/ingest/__init__.py @@ -2,8 +2,8 @@ Reads upstream catalogs (Wikipedia list pages today, vendor product pages later), normalizes each row into a TechAPI-shaped JSON record, and writes -draft records into a TechAPI checkout. A companion CI workflow opens a PR -against TechAPI so curators only review. +draft records into a TechAPI checkout. ``ingest.yml`` uploads the result as a +patch; a maintainer opens the TechAPI PR from it by hand. Entry point: ``python -m app.ingest --category cpu --limit 5``. """ diff --git a/app/ingest/__main__.py b/app/ingest/__main__.py index e5a8037..6a8c3b9 100644 --- a/app/ingest/__main__.py +++ b/app/ingest/__main__.py @@ -7,7 +7,7 @@ Streams candidates from every wired source for ``--category``, dedups against the curated dataset, writes additions to ``--data-root``, and -emits a Markdown summary (used by the weekly workflow as the PR body). +emits a Markdown summary (ingest.yml puts it in the PR body it uploads). """ from __future__ import annotations diff --git a/app/ingest/enrich.py b/app/ingest/enrich.py index 194d1ac..f14cc8e 100644 --- a/app/ingest/enrich.py +++ b/app/ingest/enrich.py @@ -16,8 +16,9 @@ python -m app.ingest.enrich --source cinebench-legacy \\ --data-root ../TechAPI/data --min-year 2011 --summary enrich.md -Run output is a PR-ready Markdown summary. Designed for the weekly-ingest -workflow, but safe to run locally (respects ``--dry-run`` and ``--sleep``). +Run output is a PR-ready Markdown summary. Designed for the manually +dispatched ingest and refresh workflows, but safe to run locally (respects +``--dry-run`` and ``--sleep``). DOM note: each source's extractor is validated against live HTML on first run; adjust selectors if a site's markup drifts. Pure logic is covered by diff --git a/app/verify/baseline.py b/app/verify/baseline.py deleted file mode 100644 index 3d1d248..0000000 --- a/app/verify/baseline.py +++ /dev/null @@ -1,27 +0,0 @@ -"""Render the daily ``status.json`` aggregate as the PR comment's baseline table.""" - -from __future__ import annotations - -import json -import sys -from pathlib import Path - - -def main(path: str) -> int: - p = Path(path) - if not p.exists(): - print("_status.json not available on this ref._") - return 0 - d = json.loads(p.read_text(encoding="utf-8")) - print(f"_As of {d['generated_at']} (post-merge `dump-develop` aggregate)._\n") - print("| category | total | green | yellow | red |") - print("| --- | ---: | ---: | ---: | ---: |") - for cat, c in d["by_category"].items(): - print(f"| {cat} | {c['total']} | {c['green']} | {c['yellow']} | {c['red']} |") - t = d["totals"] - print(f"| **all** | {t['records']} | {t['green']} | {t['yellow']} | {t['red']} |") - return 0 - - -if __name__ == "__main__": - sys.exit(main(sys.argv[1])) diff --git a/app/verify/gsmarena_backfill.py b/app/verify/gsmarena_backfill.py index f8abc12..9fcf4c7 100644 --- a/app/verify/gsmarena_backfill.py +++ b/app/verify/gsmarena_backfill.py @@ -22,7 +22,7 @@ :: - python -m app.verify.gsmarena_backfill --data-root C:/Users/29/Desktop/TechAPI \\ + python -m app.verify.gsmarena_backfill --data-root ../TechAPI \\ --limit 200 --sleep 1.5 --dry-run --summary .gsmarena-backfill/summary.md """ diff --git a/app/verify/wikipedia_gpu_backfill.py b/app/verify/wikipedia_gpu_backfill.py index 6a17d11..c57d9a4 100644 --- a/app/verify/wikipedia_gpu_backfill.py +++ b/app/verify/wikipedia_gpu_backfill.py @@ -30,7 +30,7 @@ :: python -m app.verify.wikipedia_gpu_backfill \\ - --data-root C:/path/to/TechAPI --limit 200 --sleep 1.5 --dry-run + --data-root ../TechAPI --limit 200 --sleep 1.5 --dry-run """ from __future__ import annotations diff --git a/app/verify/wikipedia_smartphone_backfill.py b/app/verify/wikipedia_smartphone_backfill.py index 053da41..df8427e 100644 --- a/app/verify/wikipedia_smartphone_backfill.py +++ b/app/verify/wikipedia_smartphone_backfill.py @@ -26,7 +26,7 @@ :: python -m app.verify.wikipedia_smartphone_backfill \\ - --data-root C:/path/to/TechAPI --limit 300 --sleep 1.0 --dry-run + --data-root ../TechAPI --limit 300 --sleep 1.0 --dry-run """ from __future__ import annotations diff --git a/app/verify/wikipedia_soc_backfill.py b/app/verify/wikipedia_soc_backfill.py index dffaa48..9cb8303 100644 --- a/app/verify/wikipedia_soc_backfill.py +++ b/app/verify/wikipedia_soc_backfill.py @@ -31,7 +31,7 @@ :: python -m app.verify.wikipedia_soc_backfill \\ - --data-root C:/path/to/TechAPI --limit 200 --sleep 1.5 --dry-run + --data-root ../TechAPI --limit 200 --sleep 1.5 --dry-run """ from __future__ import annotations diff --git a/docs/DATA_PIPELINE.md b/docs/DATA_PIPELINE.md index 32b1c78..eb2bd4c 100644 --- a/docs/DATA_PIPELINE.md +++ b/docs/DATA_PIPELINE.md @@ -56,8 +56,9 @@ live-scrape benchmark sources -> commit data/ on collect/refresh -> full-dataset gate (app.validate + integrity_check.py --strict) -> Tier 0 report for the changed records -> upload the commit as collect.patch (artifact collect-refresh) - -> a maintainer opens the PR on develop from their own account with - TechAPI's .github/scripts/open-collect-pr.sh + -> a maintainer runs TechAPI's .github/scripts/open-collect-pr.sh + by hand: their account pushes the collect/* branch and TechEngineBot + opens the PR on develop ``` The integrity gate re-checks the **whole** dataset every run (not just new diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index d402555..7c11e4b 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -21,8 +21,8 @@ see "Data scope" below). - **Formats**: list `{count,next,previous,results}` (§7.4); error `{error:{code,message,request_id,documentation_url}}` (§7.5); smartphone detail per SPEC appendix C (embeds brand+soc+manufacturer + computed `score`). -- **Tests**: `pytest` is the canonical test runner. `ruff` + `mypy app scripts` - are the static gates in CI. +- **Tests**: `pytest` is the canonical test runner. `ruff check app tests` + + `mypy app` are the static gates in CI. - **NOT done**: cloud deploy (Railway/Fly need accounts) and public static hosting. Dockerfile + workflows written but not executed in the cloud. @@ -46,20 +46,22 @@ app/ FastAPI app database.py engine + get_session (SQLite or Postgres) models/ SQLModel tables: brand, soc, smartphone, gpu, cpu schemas/ Pydantic response models + serializers.py (ORM→schema, URLs) - services/scoring.py open 0–100 scoring (§8); reference-based min-max (Phase 0) - routers/ meta, brands, socs, smartphones, gpus, cpus + services/scoring/ open 0–100 scoring (§8); weights in config/scoring.yaml + routers/ meta, brands, socs, smartphones, mobile_devices (tablets, + watches, pdas), gpus, cpus, laptops, monitors, software, + websites, device_catalog errors.py §7.5 error envelope + handlers -data/ CURATED SEED DATA — singular folder names, organised by brand: - brand//.json - soc///.json - smartphone///.json - gpu////.json - cpu////.json -scripts/ - seed.py data/ → DB (recurses brand subfolders; coerces ISO dates) - validate.py schema/range/FK/slug checks (run in CI on data changes) + data_root.py finds the dataset: TECHAPI_DATA_DIR, ../data, ../TechAPI/data + seed.py dataset → DB (recurses brand subfolders; coerces ISO dates) + validate.py schema/range/FK/slug checks dump.py DB → static JSON dump (replays API in-process) → ./dump -tests/ unit/ + integration/ (conftest seeds a temp SQLite from data/) + coverage/ upstream-vs-curated gap report + ingest/ collectors: draft new records, refresh benchmarks + verify/ trust tiers 0–3, verified promotion, backfill tools +integrity_check.py whole-dataset anomaly scan (the --strict gate) +config/scoring.yaml scoring weights and reference scales +tests/ unit/ + integration/ + verify/ +site/ Astro landing page (deploy-pages.yml publishes it with the dump) docs/ SPEC.md, DATA_PIPELINE.md, DEVELOPMENT.md .github/workflows/ test.yml, validate-data.yml, deploy-pages.yml, coverage-report.yml, final-check.yml (`final-check` label -> one TechEngineBot comment), and @@ -68,15 +70,17 @@ docs/ SPEC.md, DATA_PIPELINE.md, DEVELOPMENT.md collect.patch; nothing scheduled; ADR-017) ``` -> Note: **data folders are singular** (`data/soc/…`) but **API routes are plural** -> (`/v1/socs`). Intentional. +> Note: the dataset is **not in this repo**. It lives in TechAPI's `data/` +> (`brand//.json`, `cpu////.json`, …) +> and is found through `app/data_root.py`. Data folders are singular +> (`data/soc/…`) but API routes are plural (`/v1/socs`). Intentional. ## How to run ```bash pip install -e ".[dev]" # deps already installed in this Codespace python -m app.validate # check curated data -python -m app.seed # data/ → ./techapi.db (SQLite) +python -m app.seed # dataset → ./techapi.db (SQLite) uvicorn app.main:app --reload # serve; curl http://localhost:8000/v1/smartphones/galaxy-s25 pytest -q --cov=app # tests (target >60%, currently ~98%) ruff check app tests && mypy app @@ -98,7 +102,7 @@ python -m app.dump # generate ./dump/ static tree (gitignored) ## Conventions (IMPORTANT) - **Commits**: Conventional Commits (`feat(api): …`, `data: …`, `docs: …`). - Keep commit messages clean — no tool/AI attribution trailers. Commit/push only when asked. + Keep commit messages clean and to the point. - **Data accuracy**: only real, sourced models. All brand, device, and chip records require `source_urls`. Do not fabricate unsourced "all devices" data (SPEC §1.6). diff --git a/docs/SPEC.md b/docs/SPEC.md index c6e8ce2..09fbba3 100644 --- a/docs/SPEC.md +++ b/docs/SPEC.md @@ -1517,7 +1517,7 @@ chore: bump fastapi to 0.115 1. **감지**: TechMachine #1 코멘트 (또는 사용자 직접 발견) 2. **분류**: 데이터 문제(validate 실패) / CI 워크플로 실패 / 사이트 빌드 실패 — "서비스 다운"은 해당 없음(정적 사이트라 GitHub Pages 가용성에 종속) 3. **대응**: 원인이 코드면 fix PR, 데이터면 revert 또는 후속 보정 PR -4. **사후 분석**: 심각한 경우 `.claude/worklog_*.md`에 기록 (커밋 메시지·PR·이슈에는 에이전트 흔적을 남기지 않음) +4. **사후 분석**: 심각한 경우 원인과 조치를 후속 PR 본문에 기록 --- @@ -2005,13 +2005,13 @@ TechAPI의 **핵심 목표 중 하나**. TechPicks 외에 다양한 앱·플랫 3. 레포 간 `repository_dispatch` 중계와 서브모듈 자동 bump(`notify-*`, `bump-*`)는 없앤다. 4. `verify-status`는 TechAPI `dump-develop`에 합치고, `dump-refresh`는 `dump-develop`의 수동 전체 재생성으로 대체한다. 커버리지 리포트는 주 1회, TechEngine 레포 이슈만 갱신한다. 5. 스크레이핑 워크플로(`weekly-data.yml`)는 수동 실행으로 검증한 뒤에 cron을 켠다. - 6. **예외 — TechEngineBot 최종 확인**: TechEngineBot 계정은 중요한 PR의 최종 확인자 역할로만 쓴다. 메인테이너가 PR에 `final-check` 라벨을 붙일 때만 각 GetTechAPI 레포(TechAPI, TechEngine, TechMachine, game-catalog, cpu-engineering-samples)의 `final-check.yml`이 돌고, PR 하나에 댓글 하나(재실행 시 같은 댓글 수정)만 남긴다. 검사 작업은 시크릿 없이 읽기 전용 토큰으로 돌고, `TECHENGINEBOT_TOKEN`(TechEngineBot의 fine-grained 토큰, Pull requests: read and write)은 그 댓글 작업에만 쓴다. Copilot 리뷰는 쓰지 않는다. 스케줄·push·PR 이벤트로 TechEngineBot이 움직이는 워크플로는 만들지 않는다. + 6. **예외 — TechEngineBot**: TechEngineBot 계정은 두 가지에만 쓴다. (1) PR은 TechEngineBot이 연다. 메인테이너 머신에서 사람이 직접 TechEngineBot의 gh 로그인(메인테이너 로그인과 분리된 설정 디렉터리)으로 열며, 커밋·push·머지는 메인테이너 계정으로 한다. 어떤 워크플로도 PR을 열거나 승인하지 않는다. (2) 중요한 PR의 최종 확인: 메인테이너가 PR에 `final-check` 라벨을 붙일 때만 각 GetTechAPI 레포(TechAPI, TechEngine, TechMachine, game-catalog, cpu-engineering-samples)의 `final-check.yml`이 돌고, PR 하나에 댓글 하나(재실행 시 같은 댓글 수정)만 남긴다. 검사 작업은 시크릿 없이 읽기 전용 토큰으로 돌고, `TECHENGINEBOT_TOKEN`(TechEngineBot의 fine-grained 토큰, Pull requests: read and write)은 그 댓글 작업에만 쓴다. 스케줄·push·PR 이벤트로 TechEngineBot이 움직이는 워크플로는 만들지 않는다. - **개정 (2026-11-01)**: TechAPI는 PR 검증을 자체 검증기로 직접 돌리므로, 2·4·5번을 다음으로 대체한다. - PR 댓글 워크플로(`techapi-pr-validation-comment`, `techapi-verify-comment`)와 `refresh-data`를 삭제한다. `coverage-report`는 수동 실행·읽기 전용으로 바꾼다. - - 수집·검증(`weekly-ingest`, `weekly-refresh`, `verify-network`)은 `ingest.yml`, `refresh.yml`, `verify-promote.yml`로 바꾼다. TechAPI `collect.yml`이 `workflow_dispatch`로만 호출하는 변경 생성기다. 토큰은 읽기 전용이고, 로컬 커밋을 `collect.patch`(+ `pr-body.md`, `pr.json`) 아티팩트로 올린다. PR은 메인테이너가 자기 계정으로 TechAPI `.github/scripts/open-collect-pr.sh `를 써서 연다. 워크플로는 push·PR 생성·댓글을 하지 않으며, 게이트(`app.validate`, `integrity_check.py --strict`) 결과와 Tier 0 보고서는 PR 본문에 남긴다. 덤프는 커밋하지 않는다. 실행되는 엔진은 TechAPI가 서브모듈로 고정한 TechEngine 커밋이다. + - 수집·검증(`weekly-ingest`, `weekly-refresh`, `verify-network`)은 `ingest.yml`, `refresh.yml`, `verify-promote.yml`로 바꾼다. TechAPI `collect.yml`이 `workflow_dispatch`로만 호출하는 변경 생성기다. 토큰은 읽기 전용이고, 로컬 커밋을 `collect.patch`(+ `pr-body.md`, `pr.json`) 아티팩트로 올린다. 메인테이너가 TechAPI `.github/scripts/open-collect-pr.sh `를 직접 돌리면 자기 계정으로 `collect/*` 브랜치를 push하고, PR은 TechEngineBot이 연다. 워크플로는 push·PR 생성·댓글을 하지 않으며, 게이트(`app.validate`, `integrity_check.py --strict`) 결과와 Tier 0 보고서는 PR 본문에 남긴다. 덤프는 커밋하지 않는다. 실행되는 엔진은 TechAPI가 서브모듈로 고정한 TechEngine 커밋이다. - 크롤러는 브라우저를 흉내 내지 않는 `TechEngine-*` User-Agent로 자신을 밝힌다. 안티봇 챌린지는 "알 수 없음"으로 두고 `verified` 승급 근거로 쓰지 않는다. 검색엔진 결과 스크레이핑(GSMArena 후보 탐색용 DuckDuckGo/Bing)은 없앤다. - 모든 action은 GitHub 소유 action을 SHA로 고정해 쓴다. TechEngineBot 최종 확인(6번)은 TechAPI 쪽에서 관리한다. -- **트레이드오프**: 수집 PR은 메인테이너가 패치로 직접 열어야 한다. 대신 그 PR에서는 다른 PR처럼 검사가 돌고, 게이트는 패치를 올리기 전에 같은 작업 안에서 끝낸다. 포크 PR에는 토큰이 읽기 전용이라 댓글 대신 Job Summary로 결과를 남긴다. Projects 보드 필드 자동 입력(`pr-metadata`)은 없애고 GitHub Projects 내장 Auto-add를 쓴다. +- **트레이드오프**: 수집 PR은 자동으로 열리지 않고, 메인테이너가 헬퍼를 직접 돌려야 TechEngineBot이 연다. 대신 그 PR에서는 다른 PR처럼 검사가 돌고, 게이트는 패치를 올리기 전에 같은 작업 안에서 끝낸다. 포크 PR에는 토큰이 읽기 전용이라 댓글 대신 Job Summary로 결과를 남긴다. Projects 보드 필드 자동 입력(`pr-metadata`)은 없애고 GitHub Projects 내장 Auto-add를 쓴다. - **필요 설정**: 없음. 워크플로가 PR을 만들지 않으므로 Settings → Actions → General의 "Allow GitHub Actions to create and approve pull requests"는 모든 레포에서 끈 채로 둔다. ### ADR-018: 사진·로고는 Wikimedia Commons 파일만 — 제3자 hotlink 제거 diff --git a/integrity_check.py b/integrity_check.py index f8db27c..95d49da 100644 --- a/integrity_check.py +++ b/integrity_check.py @@ -9,9 +9,12 @@ python integrity_check.py [DATA_ROOT] [--strict] [--hard-report PATH] +Without ``DATA_ROOT`` the scan reads the directory ``app.data_root`` resolves: +``$TECHAPI_DATA_DIR``, else ``../data``, else ``../TechAPI/data``. + By default it prints every flagged item and exits 0 (human-review mode). With ``--strict`` it additionally exits non-zero when any *hard* anomaly is found — -unambiguous corruption that must block the weekly refresh PR: duplicate slugs, +unambiguous corruption that must block a collection or refresh PR: duplicate slugs, slug/filename mismatches, and physically-impossible single>multi benchmarks. The statistical cross-source/era outliers stay advisory (a heterogeneous catalog of server + desktop + mobile parts legitimately produces many ratio outliers), so @@ -24,6 +27,7 @@ from __future__ import annotations import os, json, math, re, statistics, sys from app.categories import CATEGORIES +from app.data_root import get_data_root # Em-dash etc. in section headers must not crash on legacy consoles (e.g. cp949). try: @@ -45,9 +49,9 @@ if _only_index >= 0: del _argv[_only_index:_only_index + 2] _positional = [a for a in _argv if not a.startswith("-")] -ROOT = _positional[0] if _positional else r"C:\Users\29\Desktop\TechAPI\data" +ROOT = _positional[0] if _positional else str(get_data_root()) -# Hard anomalies block the weekly gate under --strict; soft ones are review-only. +# Hard anomalies block the gate under --strict; soft ones are review-only. HARD: list[str] = [] def hard(msg: str) -> None: HARD.append(msg) diff --git a/passmark_ids.json b/passmark_ids.json deleted file mode 100644 index 7e45324..0000000 --- a/passmark_ids.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "Intel Core i7-11700": "3947", - "Intel Core i9-11900": "4245", - "Intel Core i5-12500": "4675", - "Intel Core i5-12600": "4688", - "AMD Ryzen 7 3800X": "3499", - "Intel Processor N100": "5157" -} diff --git a/pyproject.toml b/pyproject.toml index a52bfee..269dd46 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,7 +32,7 @@ dev = [ [project.urls] Homepage = "https://techapi.dev" Repository = "https://github.com/GetTechAPI/TechEngine" -Data = "https://github.com/Seungpyo1007/TechAPI" +Data = "https://github.com/GetTechAPI/TechAPI" [build-system] requires = ["setuptools>=68"] diff --git a/tests/unit/test_bot_coverage.py b/tests/unit/test_bot_coverage.py index 557c113..99564a0 100644 --- a/tests/unit/test_bot_coverage.py +++ b/tests/unit/test_bot_coverage.py @@ -5,8 +5,7 @@ import pytest -from app import dump_check -from app.categories import CATEGORIES, COLLECTIONS +from app.categories import CATEGORIES from app.verify import cli, offline from app.verify.common import CATEGORIES as VERIFY_CATEGORIES from app.verify.common import Record @@ -58,54 +57,6 @@ def write(path: Path, data): path.write_text(json.dumps(data), encoding="utf-8") -def fixture_dump(tmp_path): - collections = {} - for category, resource in COLLECTIONS.items(): - write(tmp_path / "data" / category / "sample.json", {"slug": "sample"}) - write(tmp_path / "site/public/v1" / resource / "index.json", - {"count": 1, "results": [{"slug": "sample"}]}) - collections[resource] = {"count": 1} - write(tmp_path / "site/public/v1/index.json", {"collections": collections}) - - -def test_dump_checks_parse_manifest_and_indices(monkeypatch, tmp_path): - fixture_dump(tmp_path) - changed = "site/public/v1/laptops/sample/index.json" - write(tmp_path / changed, {"slug": "sample"}) - monkeypatch.setattr(dump_check.subprocess, "run", lambda *a, **kw: - SimpleNamespace(stdout=changed + "\n")) - assert dump_check.check_dump(tmp_path, "base") == [] - (tmp_path / changed).write_text("{broken", encoding="utf-8") - assert any(changed in e for e in dump_check.check_dump(tmp_path, "base")) - write(tmp_path / changed, {}) - write(tmp_path / "site/public/v1/laptops/index.json", {"count": 2, "results": []}) - assert any("laptops: index" in e for e in dump_check.check_dump(tmp_path, "base")) - write(tmp_path / "site/public/v1/index.json", {"collections": {}}) - assert sum("manifest count" in e for e in dump_check.check_dump(tmp_path, "base")) == 13 - - -def test_dump_deletion_still_checks_counts(monkeypatch, tmp_path): - fixture_dump(tmp_path) - replies = iter(["", "site/public/v1/laptops/index.json\n"]) - monkeypatch.setattr(dump_check.subprocess, "run", lambda *a, **kw: - SimpleNamespace(stdout=next(replies))) - (tmp_path / "site/public/v1/laptops/index.json").unlink() - assert any("laptops" in e for e in dump_check.check_dump(tmp_path, "base")) - - -def test_dump_no_changes_skips_missing_dump(monkeypatch, tmp_path): - monkeypatch.setattr(dump_check.subprocess, "run", lambda *a, **kw: - SimpleNamespace(stdout="")) - assert dump_check.check_dump(tmp_path, "base") == [] - - -def test_scope_includes_all_records_and_base(tmp_path): - fixture_dump(tmp_path) - assert dump_check.scope(tmp_path, "develop", "abcdef1234") == ( - "13/13 categories ? 13 records ? diff base develop@abcdef1" - ) - - def test_pr_base_excludes_prior_develop_changes_and_release_includes_them(monkeypatch, tmp_path): def git(*args): return subprocess.run(["git", *args], cwd=tmp_path, check=True,