KiCadRoutingTools

#!/usr/bin/env python3
"""Fetch set30 .kicad_pcb sources listed in manifest_set30.json (raw download).

set30 is the dense-parallel-bus batch (2026-09-29): 21 boards whose
difficulty is a wide memory or CPU bus rather than a fine-pitch part count --
six DDR SoC boards (one LPDDR4, four DDR3, one DDR2), an FMC SDRAM+NAND
module, a PCIe x16 backplane, and 14 CPLD/CPU/DRAM boards for retro machines
(Amiga, 68000/68030, Mac, Apple II). 15 are 4-layer and 6 are 2-layer.

Two-layer DDR does not exist among the open KiCad boards that could be
found (every DDR SoC board is 4+ layers), so the six 2-layer members are
the densest 2-layer bus boards instead: pistormx_1k2 (32-bit data / 24-bit
address through a fine-pitch CPLD), mcd_pcb and computie_68kboard (68000
SBCs), and three DRAM boards (a500_2meg_chip_ram, amigaa500dram,
macclassicramcard).

Candidates came from GitHub topic/code search and web leads, were gated by
validate_candidate.py (all 21 PASS), and pinned to the commit they were
validated at -- every raw_url here is SHA-pinned so a later push to a
branch cannot change the fetched bytes.

Dedupe was three-layered. dedupe_score.py against the 456-board corpus
scored the admitted boards at most 0.239 (sam9g25 vs set12's
allwinner_a13_som: same designer, different SoC and board), and at most
0.113 pairwise among the 21, all under the 0.30 bar. A lineage check
compared the schematic-symbol UUIDs and footprint uuid/tstamp values that
footprints carry, which caught a duplicate the name/footprint score missed
(Bogdan8266's copy of the Allwinner H616 devboard, dropped; Kononenko-K's
is the one admitted); no admitted board shares lineage with the corpus
or with another admitted board. The lineage check is blind to a board with
zero symbol UUIDs (pcie_backplane_x16 is one), which is why the score and
repo-slug checks also ran. Only one board per repo is admitted where a
repo ships several near-identical variants.

Four boards ship a .kicad_dru (h616_lpddr4, imx6ulz_4l, sam9g25, cider):
h616's per-netclass DDR skew limits, imx6ulz_4l's length-window rules,
sam9g25's `(layer outer)` rules conditioned on `A.insideCourtyard`, and
cider's JLCPCB 4-layer rule set.

Downloads each board and its sibling project files -- the .kicad_pro (DRC floor,
#441) and the .kicad_dru (per-layer clearance rules, #498) --
into $STRESS_DIR/sources/github_set30/. After fetching, run `bash prep_set30.sh`
(needs KiCad's bundled python / pcbnew) to produce boards_set30/ (routed
reference) + boards_unrouted_set30/ (stripped).

  python3 fetch_set30.py
  bash prep_set30.sh

Auto-generated by assemble_corpus.py.
"""
import json
import os
import shutil
import subprocess
import sys
import tempfile
import zipfile
from pathlib import Path

HERE = Path(__file__).resolve().parent
STRESS = Path(os.environ.get("STRESS_DIR", str(Path.home() / "Documents/kicad_stress_test")))
MANIFEST = HERE / "manifest_set30.json"


def fetch_archive(b, dest):
    """Board shipped as a .zip (e.g. attached to a GitHub issue) rather than a
    raw .kicad_pcb URL. Download the archive, extract `archive_member` as the
    board, and take its siblings from the SAME directory inside the archive --
    the .kicad_pro (DRC floor, #441) and .kicad_dru (per-layer clearance, #498)
    travel with it. Returns True on success.
    """
    with tempfile.TemporaryDirectory() as td:
        zp = Path(td) / "src.zip"
        r = subprocess.run(["curl", "-sL", "--fail", b["archive_url"], "-o", str(zp)],
                           capture_output=True)
        if r.returncode != 0 or not zp.exists() or zp.stat().st_size == 0:
            return False
        try:
            with zipfile.ZipFile(zp) as z:
                z.extractall(Path(td) / "x")
        except zipfile.BadZipFile:
            return False
        member = Path(td) / "x" / b["archive_member"]
        if not member.exists():
            return False
        shutil.copy(member, dest)
        for ext in (".kicad_pro", ".kicad_dru"):
            sib = member.with_suffix(ext)
            if sib.exists():
                shutil.copy(sib, dest.with_suffix(ext))
        return True


def main():
    boards = json.loads(MANIFEST.read_text())
    out_dir = STRESS / "sources/github_set30"
    out_dir.mkdir(parents=True, exist_ok=True)
    ok = 0
    for b in boards:
        dest = out_dir / Path(b["file"]).name
        if b.get("archive_url"):
            if fetch_archive(b, dest):
                ok += 1
                print(f"  OK  {b['repo']:42} {dest.stat().st_size // 1024}KB  "
                      f"[{b.get('tier','?')}] (archive)")
            else:
                print(f"  FAIL {b['repo']}  <- {b['archive_url']}")
            continue
        r = subprocess.run(["curl", "-sL", "--fail", b["raw_url"], "-o", str(dest)],
                           capture_output=True)
        if r.returncode != 0 or not dest.exists() or dest.stat().st_size == 0:
            print(f"  FAIL {b['repo']}  <- {b['raw_url']}")
            continue
        # siblings: never drop them. Without the .kicad_pro a board resolves its
        # DRC floor from the STOCK netclass (#441); without the .kicad_dru every
        # routing step and check_drc lose the per-layer/conditioned clearance
        # rules that OUTRANK --clearance (#498). A 404 under --fail can still
        # leave a zero-byte file, which reads as "rules present, none" -- drop it.
        for ext in (".kicad_pro", ".kicad_dru"):
            sib = dest.with_suffix(ext)
            subprocess.run(["curl", "-sL", "--fail",
                            b["raw_url"][: -len(".kicad_pcb")] + ext,
                            "-o", str(sib)], capture_output=True)
            if sib.exists() and sib.stat().st_size == 0:
                sib.unlink()
        ok += 1
        print(f"  OK  {b['repo']:42} {dest.stat().st_size // 1024}KB  [{b.get('tier','?')}]")
    print(f"\n{ok}/{len(boards)} set30 sources -> {out_dir}")
    return 0 if ok == len(boards) else 1


if __name__ == "__main__":
    sys.exit(main())