Add schema+file validator; pin down fs-web-stream as ad icons
Why: - schema 必须能自动校验,否则后续放量无法防腐。现在 scripts/validate.py 对全部 metadata.json 做两层检查(schema + 本地文件 sha256),跑一次 即可对全量数据签收;10/10 项目已通过。 - docs/sources/oshwhub.md 之前把 fs-web-stream.jlc.com 标为"工程源待查", 排查后确认那些 URL 全部是嘉立创服务侧栏/推广图标,与项目无关。 image.lceda.cn/attachments/ 是项目附件的唯一入口,现在调研文档闭合。 What: - scripts/validate.py: jsonschema 校验 + optional --check-files 核 sha256 - pyproject.toml: 加 jsonschema>=4.26 依赖 - docs/sources/oshwhub.md: fs-web-stream 归类为推广资源(已排除),附 context 证据 - log.md: 本次会话记录 Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
99
scripts/validate.py
Normal file
99
scripts/validate.py
Normal file
@@ -0,0 +1,99 @@
|
||||
"""Validate all data/raw/<source>/<project>/metadata.json against project.schema.json.
|
||||
|
||||
Usage:
|
||||
uv run python scripts/validate.py # 校验全部
|
||||
uv run python scripts/validate.py data/raw/oshwhub # 指定子目录
|
||||
uv run python scripts/validate.py --check-files # 也核对本地文件是否存在+sha256 一致
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import jsonschema
|
||||
|
||||
REPO = Path(__file__).resolve().parent.parent
|
||||
SCHEMA_PATH = REPO / "schemas" / "project.schema.json"
|
||||
|
||||
|
||||
def iter_metadata(roots: list[Path]):
|
||||
for root in roots:
|
||||
for meta in root.rglob("metadata.json"):
|
||||
yield meta
|
||||
|
||||
|
||||
def sha256_of(path: Path, chunk: int = 1 << 15) -> str:
|
||||
h = hashlib.sha256()
|
||||
with open(path, "rb") as f:
|
||||
while True:
|
||||
b = f.read(chunk)
|
||||
if not b:
|
||||
break
|
||||
h.update(b)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def check_files(meta_path: Path, meta: dict) -> list[str]:
|
||||
"""Return list of errors for the file-presence / hash check."""
|
||||
errs: list[str] = []
|
||||
proj_dir = meta_path.parent
|
||||
for f in meta.get("files", []):
|
||||
rel = f.get("path")
|
||||
if not rel:
|
||||
continue # URL-only entry, ok
|
||||
p = proj_dir / rel
|
||||
if not p.exists():
|
||||
errs.append(f"missing file: {rel}")
|
||||
continue
|
||||
want = f.get("sha256")
|
||||
if want:
|
||||
got = sha256_of(p)
|
||||
if got != want:
|
||||
errs.append(f"sha256 mismatch for {rel}: {got} != {want}")
|
||||
return errs
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
ap = argparse.ArgumentParser(description="schema + file integrity check")
|
||||
ap.add_argument(
|
||||
"roots",
|
||||
nargs="*",
|
||||
type=Path,
|
||||
default=[REPO / "data" / "raw"],
|
||||
help="目录,递归找 metadata.json(默认 data/raw)",
|
||||
)
|
||||
ap.add_argument("--check-files", action="store_true", help="验证本地文件存在且 sha256 一致")
|
||||
args = ap.parse_args(argv)
|
||||
|
||||
schema = json.loads(SCHEMA_PATH.read_text())
|
||||
validator = jsonschema.Draft202012Validator(schema)
|
||||
|
||||
ok = 0
|
||||
bad = 0
|
||||
for meta_path in iter_metadata(args.roots):
|
||||
meta = json.loads(meta_path.read_text())
|
||||
rel = meta_path.relative_to(REPO)
|
||||
|
||||
errors = [f"schema: {e.message} at {'/'.join(map(str, e.path))}" for e in validator.iter_errors(meta)]
|
||||
if args.check_files:
|
||||
errors += [f"file: {m}" for m in check_files(meta_path, meta)]
|
||||
|
||||
if errors:
|
||||
bad += 1
|
||||
print(f"FAIL {rel}")
|
||||
for e in errors:
|
||||
print(f" - {e}")
|
||||
else:
|
||||
ok += 1
|
||||
|
||||
total = ok + bad
|
||||
print(f"\n{ok}/{total} passed")
|
||||
return 0 if bad == 0 else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user