Files

102 lines
4.2 KiB
Python

# -*- coding: utf-8 -*-
"""一键复查:把所有院校公告栏重扫一遍,只增不减,然后导出最新通知清单。
python refresh.py # 全流程
python refresh.py --workers 12 # 调并发
python refresh.py --skip-probe # 入口已探测过,只扫新通知
跑得比单跑快的办法是分片并发(六路各自写分片再合并),脚本会自动用这个方式:
python refresh.py --split 6
"""
import argparse
import glob
import os
import subprocess
import sys
import time
HERE = os.path.dirname(os.path.abspath(__file__))
ROOT = os.path.dirname(HERE)
PY = sys.executable
def run(args, quiet=False):
cmd = [PY, os.path.join(HERE, "fetch_official.py")] + args
print("$", " ".join(args), flush=True)
p = subprocess.run(cmd, cwd=HERE)
if p.returncode != 0:
print("!! 非零退出:", p.returncode, " ".join(args), flush=True)
return p.returncode
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--workers", type=int, default=16)
ap.add_argument("--split", type=int, default=0, help="分几路并发扫,0=单路")
ap.add_argument("--skip-probe", action="store_true")
ap.add_argument("--days", type=int, default=45)
ap.add_argument("--limit", type=int, default=0, help="只处理前 N 所,用于自检")
ap.add_argument("--deep", action="store_true", help="先从已存正文反推缺入口院校的栏目页")
ap.add_argument("--no-attach", action="store_true", help="跳过附件抓取")
ap.add_argument("--sub", type=int, default=4, help="每校再跟进几个子栏目页,0=只看入口首屏")
a = ap.parse_args()
t0 = time.time()
print("=== 复查开始 %s ===" % time.strftime("%Y-%m-%d %H:%M:%S"))
run(["--derive"])
if a.deep:
# 有公告却没探到入口的院校,先从已存正文里反推栏目页,再统一探测。
cmd = [PY, os.path.join(HERE, "derive_from_body.py"), "--apply"]
print("$", "derive_from_body.py --apply", flush=True)
subprocess.run(cmd, cwd=HERE)
if not a.skip_probe:
run(["--probe", "--workers", str(a.workers)])
os.makedirs(os.path.join(HERE, "抓取分片"), exist_ok=True)
if a.split and a.split > 1:
procs = []
for i in range(a.split):
cmd = [PY, os.path.join(HERE, "fetch_official.py"), "--sweep",
"--shard", "%d/%d" % (i, a.split), "--workers", str(max(4, a.workers // a.split)),
"--out", os.path.join("抓取分片", "auto_%d.jsonl" % i)]
if a.skip_probe:
cmd.append("--only-empty")
cmd += ["--sub", str(a.sub)]
if a.limit:
cmd += ["--limit", str(a.limit)]
procs.append((subprocess.Popen(cmd, cwd=HERE), i))
for p, i in procs:
p.wait()
shards = sorted(glob.glob(os.path.join(HERE, "抓取分片", "auto_*.jsonl")))
run(["--ingest"] + [os.path.relpath(s, HERE) for s in shards])
else:
run(["--sweep", "--workers", str(a.workers)])
run(["--consolidate"]) # 合并重复必须先于回填,否则重复会被带进派生表
run(["--backfill"])
if not a.no_attach:
run(["--attach", "--workers", str(a.workers)])
run(["--report", "--days", str(a.days)])
att = os.path.join(ROOT, "公告抽取_2027附件字段.csv")
if os.path.exists(att):
cmd = [PY, os.path.join(HERE, "merge_att_fields.py")]
print("$", "merge_att_fields.py", flush=True)
subprocess.run(cmd, cwd=HERE)
for extra in ("reconcile_plan27.py", "report_coverage.py"):
ep = os.path.join(HERE, extra)
if os.path.exists(ep):
print("$", extra, flush=True)
subprocess.run([PY, ep], cwd=HERE)
cmd = [PY, os.path.join(HERE, "verify_db.py")]
print("$", "verify_db.py", flush=True)
subprocess.run(cmd, cwd=HERE)
print("=== 完成,用时 %.0f 秒 ===" % (time.time() - t0))
print("清单见 择校/最新通知.md 与 最新通知.csv;库里 notices_all 为最新。")
print("官方 0856 层需单独刷新时跑 integrate_chsi.py → align_chsi.py → export_api.py"
"(存档 JSON 有新增才需要)")
if __name__ == "__main__":
main()