Add EasyEDA Pro 2.x legacy source ingestion (5/5 batch closure)

补齐前一批失败的 2 个 legacy Pro 项目（立创·泰山派 RK3566、立创·梁山派），打通 Pro 2.x 旧版工程的源抓取链路。结合上一 commit 的 modern Pro 3.x 路径，本仓库 5/5 Pro 项目 EPRO2/dataStr 全部端到端打通。 Pro 2.x 与 Pro 3.x 是两个完全不同的存储模型： - Pro 3.x：git-style branch + linear history chain，AES-128-GCM 加密的 EPRO2 增量消息流，按 history 重放（已在前一 commit 打通） - Pro 2.x：无 branch / 无 history。文档以 EasyEDA Std plaintext dataStr 存储（同 ["DOCTYPE","SCH","1.1"] 格式），按 doc UUID 通过 /api/v2/documents/lists 批量 GET，主体无加密，只组件库走 AES Pro 2.x 抓取链由 HAR (tmp/prodownload3.har, 178 请求) 反推： GET /api/v4/projects/<P> → boards: [{sch, pcb, name}] GET /api/projects/<P>/ticket?uuid=&g_ticket=-1 → 完整项目 manifest POST /api/schematic/lists {uuids:[<sch>]} → sort: [{uuid:<sheet>}] POST /api/v2/documents/lists {uuids,docType:1} → schematic plaintext POST /api/v2/documents/lists {uuids,docType:3} → PCB plaintext POST /api/coppers/search {paths} → 铺铜层 POST /api/textpath/search {paths,project_uuid}→ 字体/文字 POST /api/v2/resources/search {hash,project_uuid} → BLOB 图片实现： - crawlers/oshwhub/crawler.py: - fetch_pro_source() refactor 成 dispatcher，先 GET project meta 检查 branch_uuid，null 即旧版走 _fetch_pro_legacy()，非空走 _fetch_pro_modern() - _fetch_pro_legacy() 新增（按上面 9 步流程拉所有 doc + 辅助层） - _pro_post_json() POST helper（与 _pro_get_json 对称） - schemas/project.schema.json: source_format enum 加 easyeda-pro-legacy - docs/sources/easyeda_pro_source.md rev 4: §1.1 旧版 vs 新版判别表更新、 §2.7 新增旧版抓取流程 + 实测数据落盘约定（旧版）： source/ticket.json 完整 manifest source/<sheet_uuid>.json 每张原理图（含 dataStr） source/pcb_<pcb_uuid>.json 每块 PCB source/coppers.json/textpath.json/blobs.json 辅助 PCB 层资源 source/manifest.json 索引实测：立创·梁山派 editor=2.1.30, 2 sheets+1 pcb, 1.0 MB, 78 sym/191 fp/128 dev 立创·泰山派 RK3566 editor=2.1.40, 29 sheets+1 pcb, 0.8 MB, 299 sym/524 fp/295 dev 旧版项目体量比新版小两个数量级（梁山派 1 MB vs RK3576 66 MB）—— 没有增量 history，组件库走单独端点，本身就是当前快照。 5/5 Pro 项目终极汇总： X86 主板 easyeda-pro 3.2.15 7374 docs / 481 MB 泰山派 RK3566 easyeda-pro-legacy 2.1.40 30 docs / 0.8 MB 梁山派 easyeda-pro-legacy 2.1.30 3 docs / 1.0 MB 220V 桌面电源 easyeda-pro 3.2.69 771 docs / 26 MB ESP-VoCat easyeda-pro 3.2.91 278 docs / 7.5 MB 共 8456 docs / ~516 MB plain。 Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-04-28 21:59:25 +08:00
parent 3282a028c4
commit c6279bff08
48 changed files with 2603 additions and 22 deletions
--- a/crawlers/oshwhub/crawler.py
+++ b/crawlers/oshwhub/crawler.py
@@ -428,15 +428,34 @@ def fetch_pro_source(
    proj_dir: Path,
    sleep: float = SLEEP_PRO,
 ) -> dict:
-    """Fetch EasyEDA Pro project source: full history chain, AES-GCM decrypted,
-    gunzipped, and partitioned into per-document EPRO2 streams.
+    """Dispatcher: pick modern (3.x branch+EPRO2) vs legacy (2.x v2/documents/lists)
+    based on whether project meta contains a non-null branch_uuid.
+
+    Pro 3.x stores in git-style branch+history with AES-encrypted EPRO2 streams;
+    Pro 2.x predates that and uses Std-style per-doc dataStr served from
+    /api/v2/documents/lists. See docs/sources/easyeda_pro_source.md §1.1.
+    """
+    proj = _pro_get_json(pro_client, f"{PRO_API}/projects/{project_uuid}", project_uuid)
+    time.sleep(sleep)
+    if proj.get("branch_uuid"):
+        return _fetch_pro_modern(pro_client, project_uuid, proj, proj_dir, sleep)
+    return _fetch_pro_legacy(pro_client, project_uuid, proj, proj_dir, sleep)
+
+
+def _fetch_pro_modern(
+    pro_client: httpx.Client,
+    project_uuid: str,
+    proj: dict,
+    proj_dir: Path,
+    sleep: float = SLEEP_PRO,
+) -> dict:
+    """Modern Pro 3.x fetcher: full history chain, AES-GCM decrypted, gunzipped,
+    and partitioned into per-document EPRO2 streams.

    Side effects under ``proj_dir``:
      - source/structure.json     — project document tree (boards/schematics/sheets/pcbs/...)
      - source/<doc_uuid>.epro2   — one file per document, raw EPRO2 messages (newline-separated)
      - source/manifest.json      — per-doc index + chain summary
-
-    Returns dict matching the shape `fetch_std_source` returns.
    """
    import gzip
    from collections import OrderedDict
@@ -445,14 +464,8 @@ def fetch_pro_source(
    src_dir = proj_dir / "source"
    src_dir.mkdir(parents=True, exist_ok=True)

-    # 1. project meta -> branch_uuid + editor_version fallback
-    proj = _pro_get_json(pro_client, f"{PRO_API}/projects/{project_uuid}", project_uuid)
-    branch_uuid = proj.get("branch_uuid")
-    if not branch_uuid:
-        raise RuntimeError(f"no branch_uuid in project meta for {project_uuid}")
-    # Some projects' DOCHEAD payloads lack `editVersion`; project meta has `editorVersion`
+    branch_uuid = proj["branch_uuid"]
    project_editor_version = proj.get("editorVersion")
-    time.sleep(sleep)

    # 2. branch meta -> head history_uuid
    branch = _pro_get_json(
@@ -597,6 +610,218 @@ def fetch_pro_source(
    }


+def _pro_post_json(
+    client: httpx.Client,
+    url: str,
+    project_uuid: str,
+    body: dict,
+) -> object:
+    """POST a pro.lceda.cn endpoint with `path` header, validate envelope."""
+    r = client.post(
+        url,
+        json=body,
+        headers={"path": project_uuid, "Content-Type": "application/json"},
+    )
+    r.raise_for_status()
+    j = r.json()
+    if not j.get("success"):
+        raise RuntimeError(f"Pro API failed (POST {url}): {j}")
+    return j["result"]
+
+
+def _fetch_pro_legacy(
+    pro_client: httpx.Client,
+    project_uuid: str,
+    proj: dict,
+    proj_dir: Path,
+    sleep: float = SLEEP_PRO,
+) -> dict:
+    """Legacy Pro 2.x fetcher: project meta has `boards: [{sch, pcb, name}]` and
+    no branch model. Documents are fetched via `/api/v2/documents/lists` (Std-style
+    plaintext dataStr); resources/coppers/textpath/blobs come from supplementary
+    POST endpoints. Reverse-engineered from HAR `tmp/prodownload3.har`
+    (2026-04-28); see docs/sources/easyeda_pro_source.md §1.1.
+
+    Side effects under ``proj_dir``:
+      - source/ticket.json                     — full project manifest (counts of all asset types)
+      - source/<sheet_uuid>.json               — schematic sheet content (docType=1)
+      - source/pcb_<pcb_uuid>.json             — PCB content (docType=3)
+      - source/coppers.json                    — copper pour data (if any)
+      - source/textpath.json                   — text path / font data (if any)
+      - source/blobs.json                      — embedded image blobs (if any)
+      - source/manifest.json                   — index across all of the above
+    """
+    src_dir = proj_dir / "source"
+    src_dir.mkdir(parents=True, exist_ok=True)
+
+    boards = proj.get("boards") or []
+    if not boards:
+        raise RuntimeError(f"legacy project {project_uuid} has no boards[] in meta")
+    project_editor_version = proj.get("editorVersion")
+
+    # 1. ticket — full manifest (counts of every asset type the project owns)
+    ticket = pro_client.get(
+        f"https://pro.lceda.cn/api/projects/{project_uuid}/ticket",
+        params={"uuid": project_uuid, "g_ticket": "-1"},
+        headers={"path": project_uuid},
+    )
+    ticket.raise_for_status()
+    ticket_j = ticket.json()
+    if not ticket_j.get("success"):
+        raise RuntimeError(f"ticket endpoint failed: {ticket_j}")
+    manifest_ticket = ticket_j["result"]
+    (src_dir / "ticket.json").write_text(
+        json.dumps(manifest_ticket, ensure_ascii=False, indent=2), encoding="utf-8"
+    )
+    time.sleep(sleep)
+
+    doc_metas: list[dict] = []
+
+    # 2. schematic containers -> sheet UUIDs via /api/schematic/lists
+    sch_container_uuids = [b["sch"] for b in boards if b.get("sch")]
+    sheet_uuids: list[str] = []
+    if sch_container_uuids:
+        containers = _pro_post_json(
+            pro_client,
+            "https://pro.lceda.cn/api/schematic/lists",
+            project_uuid,
+            {"uuids": sch_container_uuids},
+        )
+        if isinstance(containers, list):
+            for c in containers:
+                for s in c.get("sort") or []:
+                    su = s.get("uuid")
+                    if su:
+                        sheet_uuids.append(su)
+        time.sleep(sleep)
+
+    # 3. schematic sheets via documents/lists docType=1 (plaintext dataStr per sheet)
+    if sheet_uuids:
+        sheets = _pro_post_json(
+            pro_client,
+            "https://pro.lceda.cn/api/v2/documents/lists",
+            project_uuid,
+            {"uuids": sheet_uuids, "docType": 1},
+        )
+        for s in (sheets or []):
+            doc_uuid = s["uuid"]
+            local_rel = f"source/{doc_uuid}.json"
+            text = json.dumps(s, ensure_ascii=False, separators=(",", ":"))
+            (proj_dir / local_rel).write_text(text, encoding="utf-8")
+            doc_metas.append({
+                "doc_uuid": doc_uuid,
+                "docType": 1,
+                "path": local_rel,
+                "size": len(text.encode("utf-8")),
+                "sha256": hashlib.sha256(text.encode("utf-8")).hexdigest(),
+            })
+        time.sleep(sleep)
+
+    # 4. PCB documents via documents/lists docType=3
+    pcb_uuids = [b["pcb"] for b in boards if b.get("pcb")]
+    if pcb_uuids:
+        pcbs = _pro_post_json(
+            pro_client,
+            "https://pro.lceda.cn/api/v2/documents/lists",
+            project_uuid,
+            {"uuids": pcb_uuids, "docType": 3},
+        )
+        for p in (pcbs or []):
+            doc_uuid = p["uuid"]
+            local_rel = f"source/pcb_{doc_uuid}.json"
+            text = json.dumps(p, ensure_ascii=False, separators=(",", ":"))
+            (proj_dir / local_rel).write_text(text, encoding="utf-8")
+            doc_metas.append({
+                "doc_uuid": doc_uuid,
+                "docType": 3,
+                "path": local_rel,
+                "size": len(text.encode("utf-8")),
+                "sha256": hashlib.sha256(text.encode("utf-8")).hexdigest(),
+            })
+        time.sleep(sleep)
+
+    # 5. supplementary PCB layer assets — coppers / textpath / resources (blobs)
+    aux: dict[str, object] = {}
+    copper_paths = list((manifest_ticket.get("coppers") or {}).keys())
+    if copper_paths:
+        coppers = _pro_post_json(
+            pro_client,
+            "https://pro.lceda.cn/api/coppers/search",
+            project_uuid,
+            {"paths": copper_paths},
+        )
+        (src_dir / "coppers.json").write_text(
+            json.dumps(coppers, ensure_ascii=False), encoding="utf-8"
+        )
+        aux["coppers_count"] = len(coppers) if isinstance(coppers, list) else 0
+        time.sleep(sleep)
+
+    textpath_paths = list((manifest_ticket.get("textpath") or {}).keys())
+    if textpath_paths:
+        textpath = _pro_post_json(
+            pro_client,
+            "https://pro.lceda.cn/api/textpath/search",
+            project_uuid,
+            {
+                "paths": textpath_paths,
+                "project_uuid": project_uuid,
+                "path": project_uuid,
+            },
+        )
+        (src_dir / "textpath.json").write_text(
+            json.dumps(textpath, ensure_ascii=False), encoding="utf-8"
+        )
+        aux["textpath_count"] = len(textpath) if isinstance(textpath, list) else 0
+        time.sleep(sleep)
+
+    blob_hashes = list((manifest_ticket.get("blobs") or {}).keys())
+    if blob_hashes:
+        blobs = _pro_post_json(
+            pro_client,
+            "https://pro.lceda.cn/api/v2/resources/search",
+            project_uuid,
+            {"hash": blob_hashes, "project_uuid": project_uuid},
+        )
+        (src_dir / "blobs.json").write_text(
+            json.dumps(blobs, ensure_ascii=False), encoding="utf-8"
+        )
+        aux["blobs_count"] = len(blobs) if isinstance(blobs, list) else 0
+        time.sleep(sleep)
+
+    # 6. manifest.json — overall index
+    structure_summary = {
+        "boards": len(boards),
+        "schematic_containers": len(sch_container_uuids),
+        "schematic_sheets": len(sheet_uuids),
+        "pcbs": len(pcb_uuids),
+        "symbols": len(manifest_ticket.get("symbols") or {}),
+        "footprints": len(manifest_ticket.get("footprints") or {}),
+        "devices": len(manifest_ticket.get("devices") or {}),
+        "coppers": len(manifest_ticket.get("coppers") or {}),
+        "textpath": len(manifest_ticket.get("textpath") or {}),
+        "blobs": len(manifest_ticket.get("blobs") or {}),
+    }
+    manifest = {
+        "project_uuid": project_uuid,
+        "fetched_at": datetime.now(timezone.utc).isoformat(),
+        "editor_version": project_editor_version,
+        "boards": boards,
+        "documents": doc_metas,
+        "structure_summary": structure_summary,
+        "aux": aux,
+    }
+    (src_dir / "manifest.json").write_text(
+        json.dumps(manifest, ensure_ascii=False, indent=2), encoding="utf-8"
+    )
+
+    return {
+        "source_format": "easyeda-pro-legacy",
+        "source_path": "source/",
+        "source_documents": doc_metas,
+        "editor_version": project_editor_version,
+    }
+
+
 # ---------------------------------------------------------------------------
 # Single-project crawl
 # ---------------------------------------------------------------------------