KiCadRoutingTools

#!/usr/bin/env python3
"""Fetch set29 .kicad_pcb sources listed in manifest_set29.json (raw download).

set29 is the awesome-kicad-projects mining batch (2026-09-13): the
way2pramil/awesome-kicad-projects curated list, 71 distinct project repos,
mined end to end. 22 of those repos were ALREADY in the corpus, which is the
useful signal about the source -- it overlaps the existing corpus heavily
and the remainder is what it adds.

Of the 49 new repos, 4 ship no .kicad_pcb at all and 22 are ENTIRELY
pre-KiCad-6 (a rescue scan of all 51 alternate boards in those repos found
no v6+ revision anywhere in their trees), which validate_candidate.py
rejects for having no sibling .kicad_pro. Of the 15 that survived, one
(phodina/echo-debug-gen3) failed the >=20 routable-nets floor and one
(smartprint_h7x) classified `monster` and went to set3monster -- leaving
these 12.

A 13th, ButterStick, was dropped as a DUPLICATE that no repo-level check
could catch: the awesome list points at the org repo
butterstick-fpga/butterstick-hardware, while the corpus already holds the
identical board from the author's personal repo gregdavill/butterstick
(set2, same hardware/ButterStick_r1.0/ path, and set2's manifest predates
the short_name field so a name check would not have seen it either).
dedupe_score.py scored it 1.000 against the 443-board pre-addition corpus;
the 13 admitted boards scored 0.008 to 0.137 against the same corpus, all
well under the 0.30 bar, and 0.078 worst pairwise among themselves.
Run dedupe_score BEFORE prepping: once a candidate is in boards_setN/ it
matches its own copy and every score reads 1.000.

What the set adds beyond board count: THREE boards with a real .kicad_dru
(#498), taking the corpus from two to five, and each in a rule dialect the
existing two do not use -- piunora's netclass-conditioned diff-pair
clearances (90ohm 0.1016mm / 100ohm 0.127mm), amulet_controller's PCBWay DFM
set scoped by `(layer outer)` / `(layer inner)`, which is the ONLY corpus
board exercising the per-layer clearance path, and twonkie's NEGATIVE
edge_clearance (min -1mm) conditioned on `A.insideCourtyard('J1'|'J2')`.
Neither mez_rx nor storm_tracker carries a negative constraint, so twonkie
and amulet_controller are the first two that do.

One claim deliberately NOT made: amulet_controller and usbarmory carry user
layer nicknames ('L1 (Sig, PWR)', 'TOP'/'SIGNAL_1'/'GND_POWER_1'), which
reads like a novel parser surface and is not one -- they sit ALONGSIDE the
canonical F.Cu/In1.Cu names (the 4th token), 35 corpus boards already carry
them, and zynq7020_som uses the identical 'L1 (Sig)' style. Measured before
writing it down.

Downloads each board and its sibling project files -- the .kicad_pro (DRC floor,
#441) and the .kicad_dru (per-layer clearance rules, #498) --
into $STRESS_DIR/sources/github_set29/. After fetching, run `bash prep_set29.sh`
(needs KiCad's bundled python / pcbnew) to produce boards_set29/ (routed
reference) + boards_unrouted_set29/ (stripped).

  python3 fetch_set29.py
  bash prep_set29.sh

Auto-generated by assemble_corpus.py.
"""
import json
import os
import shutil
import subprocess
import sys
import tempfile
import zipfile
from pathlib import Path

HERE = Path(__file__).resolve().parent
STRESS = Path(os.environ.get("STRESS_DIR", str(Path.home() / "Documents/kicad_stress_test")))
MANIFEST = HERE / "manifest_set29.json"


def fetch_archive(b, dest):
    """Board shipped as a .zip (e.g. attached to a GitHub issue) rather than a
    raw .kicad_pcb URL. Download the archive, extract `archive_member` as the
    board, and take its siblings from the SAME directory inside the archive --
    the .kicad_pro (DRC floor, #441) and .kicad_dru (per-layer clearance, #498)
    travel with it. Returns True on success.
    """
    with tempfile.TemporaryDirectory() as td:
        zp = Path(td) / "src.zip"
        r = subprocess.run(["curl", "-sL", "--fail", b["archive_url"], "-o", str(zp)],
                           capture_output=True)
        if r.returncode != 0 or not zp.exists() or zp.stat().st_size == 0:
            return False
        try:
            with zipfile.ZipFile(zp) as z:
                z.extractall(Path(td) / "x")
        except zipfile.BadZipFile:
            return False
        member = Path(td) / "x" / b["archive_member"]
        if not member.exists():
            return False
        shutil.copy(member, dest)
        for ext in (".kicad_pro", ".kicad_dru"):
            sib = member.with_suffix(ext)
            if sib.exists():
                shutil.copy(sib, dest.with_suffix(ext))
        return True


def main():
    boards = json.loads(MANIFEST.read_text())
    out_dir = STRESS / "sources/github_set29"
    out_dir.mkdir(parents=True, exist_ok=True)
    ok = 0
    for b in boards:
        dest = out_dir / Path(b["file"]).name
        if b.get("archive_url"):
            if fetch_archive(b, dest):
                ok += 1
                print(f"  OK  {b['repo']:42} {dest.stat().st_size // 1024}KB  "
                      f"[{b.get('tier','?')}] (archive)")
            else:
                print(f"  FAIL {b['repo']}  <- {b['archive_url']}")
            continue
        r = subprocess.run(["curl", "-sL", "--fail", b["raw_url"], "-o", str(dest)],
                           capture_output=True)
        if r.returncode != 0 or not dest.exists() or dest.stat().st_size == 0:
            print(f"  FAIL {b['repo']}  <- {b['raw_url']}")
            continue
        # siblings: never drop them. Without the .kicad_pro a board resolves its
        # DRC floor from the STOCK netclass (#441); without the .kicad_dru every
        # routing step and check_drc lose the per-layer/conditioned clearance
        # rules that OUTRANK --clearance (#498). A 404 under --fail can still
        # leave a zero-byte file, which reads as "rules present, none" -- drop it.
        for ext in (".kicad_pro", ".kicad_dru"):
            sib = dest.with_suffix(ext)
            subprocess.run(["curl", "-sL", "--fail",
                            b["raw_url"][: -len(".kicad_pcb")] + ext,
                            "-o", str(sib)], capture_output=True)
            if sib.exists() and sib.stat().st_size == 0:
                sib.unlink()
        ok += 1
        print(f"  OK  {b['repo']:42} {dest.stat().st_size // 1024}KB  [{b.get('tier','?')}]")
    print(f"\n{ok}/{len(boards)} set29 sources -> {out_dir}")
    return 0 if ok == len(boards) else 1


if __name__ == "__main__":
    sys.exit(main())