Files
yanxuan/脚本/format_normalization_audit_final_20261003.py

186 lines
8.1 KiB
Python

from __future__ import annotations
import csv
import hashlib
import json
import os
from collections import Counter, defaultdict
from datetime import datetime, timezone
from pathlib import Path
ROOT = Path(r"D:\择校")
CATEGORIES = (
"原始来源",
"解析数据",
"数据库",
"staging",
"已入库",
"冲突",
"待补信息",
"脚本",
"审计报告",
"缓存",
"临时产物",
"重复候选",
"未知",
)
OUTPUT = ROOT / "审计报告" / "20260930_global" / "files" / "format_normalization_audit_20261003.json"
TEXT_EXTENSIONS = {".py", ".ps1", ".psm1", ".cmd", ".bat", ".sh", ".md", ".txt", ".csv", ".tsv", ".json", ".jsonl", ".yaml", ".yml", ".toml", ".ini", ".cfg", ".html", ".js", ".sql", ".log"}
STRUCTURED_EXTENSIONS = {".json", ".jsonl", ".csv", ".tsv"}
SAMPLE_PER_EXT = 3
SAMPLE_BYTES = 65536
def sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for block in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(block)
return digest.hexdigest()
def inspect_text(path: Path) -> dict:
raw_full = path.read_bytes()
raw = raw_full[:SAMPLE_BYTES]
bom = "none"
if raw.startswith(b"\xef\xbb\xbf"):
bom = "utf8"
elif raw.startswith(b"\xff\xfe") or raw.startswith(b"\xfe\xff"):
bom = "utf16"
if b"\x00" in raw[:4096] and not raw.startswith((b"\xef\xbb\xbf", b"\xff\xfe", b"\xfe\xff")):
encoding = "utf-16-le" if raw[1:2] == b"\x00" else "utf-16-be"
else:
encoding = "utf-8"
text = raw.decode(encoding, errors="replace")
if "\ufffd" in text and encoding == "utf-8":
try:
text = raw.decode("gb18030")
encoding = "gb18030"
except UnicodeDecodeError:
pass
result = {"encoding_sample": encoding, "bom": bom}
if not text or path.suffix.lower() not in STRUCTURED_EXTENSIONS:
return result
if path.suffix.lower() == ".json":
if path.stat().st_size > 5 * 1024 * 1024:
result["structured_parse"] = "json_not_checked_large"
return result
try:
json.loads(path.read_text(encoding=encoding))
result["structured_parse"] = "json_pass"
except Exception as exc:
result["structured_parse"] = "json_partial_or_fail"
result["parse_error"] = str(exc)[:300]
elif path.suffix.lower() == ".jsonl":
checked = 0
failures = 0
if path.stat().st_size <= 5 * 1024 * 1024:
text = path.read_bytes().decode(encoding, errors="replace")
for line in text.splitlines():
if not line.strip():
continue
checked += 1
try:
json.loads(line)
except Exception:
failures += 1
if checked >= 5:
break
result["structured_parse"] = "jsonl_sample_pass" if failures == 0 else "jsonl_sample_fail"
result["jsonl_sample_lines"] = checked
result["jsonl_sample_failures"] = failures
else:
first_line = next((line for line in text.splitlines() if line.strip()), "")
try:
dialect = csv.Sniffer().sniff(first_line, delimiters=",\t;|")
result["delimiter_sample"] = dialect.delimiter
except csv.Error:
result["delimiter_sample"] = "undetected"
return result
def main() -> None:
counts = defaultdict(lambda: {"file_count": 0, "logical_bytes": 0, "extensions": Counter()})
samples = []
sample_seen = Counter()
read_errors = []
large_unchecked = []
for category in CATEGORIES:
base = ROOT / category
if not base.exists():
read_errors.append({"path": str(base), "error": "category_missing"})
continue
for dirpath, dirnames, filenames in os.walk(base):
dirnames[:] = [name for name in dirnames if name not in {"__pycache__", ".pytest_cache", "node_modules"}]
for name in filenames:
path = Path(dirpath) / name
try:
stat = path.stat()
except OSError as exc:
read_errors.append({"path": str(path), "error": str(exc)})
continue
ext = path.suffix.lower() or "[no_extension]"
counts[category]["file_count"] += 1
counts[category]["logical_bytes"] += stat.st_size
counts[category]["extensions"][ext] += 1
if stat.st_size > 20 * 1024 * 1024:
large_unchecked.append({"path": str(path), "bytes": stat.st_size, "reason": "large_file_sample_only"})
sample_key = (category, ext)
if sample_seen[sample_key] >= SAMPLE_PER_EXT or ext not in TEXT_EXTENSIONS:
continue
sample_seen[sample_key] += 1
row = {"path": str(path), "category": category, "extension": ext, "bytes": stat.st_size}
try:
row.update(inspect_text(path))
except (OSError, ValueError) as exc:
row["inspect_error"] = str(exc)[:300]
samples.append(row)
category_rows = []
for category in CATEGORIES:
row = counts[category]
category_rows.append({
"category": category,
"file_count": row["file_count"],
"logical_bytes": row["logical_bytes"],
"logical_gib": round(row["logical_bytes"] / (1024 ** 3), 6),
"extensions": dict(sorted(row["extensions"].items())),
})
active_code_ext = {".py", ".ps1", ".psm1", ".cmd", ".bat", ".sh"}
scripts = [row for row in samples if row["category"] == "脚本" and row["extension"] in active_code_ext]
script_bad_encoding = [row for row in scripts if row.get("encoding_sample") not in {"utf-8", "utf-16"}]
structured_failures = [row for row in samples if row.get("structured_parse", "").endswith("fail")]
report = {
"schema_version": "format_normalization_audit.v1",
"generated_at_utc": datetime.now(timezone.utc).isoformat(),
"mode": "read_only_metadata_and_representative_content_sample",
"canonical_root": str(ROOT),
"category_count": len(CATEGORIES),
"categories": category_rows,
"sample_policy": {"sample_per_category_extension": SAMPLE_PER_EXT, "sample_bytes": SAMPLE_BYTES, "large_file_threshold_bytes": 20 * 1024 * 1024},
"samples": samples,
"script_bad_encoding_sample_count": len(script_bad_encoding),
"structured_sample_failure_count": len(structured_failures),
"large_files_sample_only_count": len(large_unchecked),
"large_files_sample_only": large_unchecked[:1000],
"read_error_count": len(read_errors),
"read_errors": read_errors[:1000],
"content_policy": {
"active_scripts_and_generated_manifests": "UTF-8 text; path rewrite batch already wrote modified scripts as UTF-8 without BOM",
"raw_sources_and_conflict_evidence": "byte-preserving; no re-encoding or overwrite",
"tabular_sources": "retain original workbook/CSV format; parser and delimiter recorded per sample",
"database": "SQLite remains the single production database; no parallel database created",
},
"verdict": "PASS" if not script_bad_encoding and not structured_failures and not read_errors else "PARTIAL",
"note": "This is a representative content audit, not a claim that every large binary or every row has been semantically re-parsed. Original source bytes remain protected.",
}
OUTPUT.parent.mkdir(parents=True, exist_ok=True)
OUTPUT.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
OUTPUT.with_suffix(OUTPUT.suffix + ".sha256").write_text(f"{sha256(OUTPUT)} {OUTPUT.name}\n", encoding="ascii")
print(json.dumps({"category_count": len(CATEGORIES), "sample_count": len(samples), "script_bad_encoding_sample_count": len(script_bad_encoding), "structured_sample_failure_count": len(structured_failures), "large_files_sample_only_count": len(large_unchecked), "verdict": report["verdict"]}, ensure_ascii=False))
if __name__ == "__main__":
main()