102 lines
4.2 KiB
Python
102 lines
4.2 KiB
Python
# -*- coding: utf-8 -*-
|
|
"""一键复查:把所有院校公告栏重扫一遍,只增不减,然后导出最新通知清单。
|
|
|
|
python refresh.py # 全流程
|
|
python refresh.py --workers 12 # 调并发
|
|
python refresh.py --skip-probe # 入口已探测过,只扫新通知
|
|
|
|
跑得比单跑快的办法是分片并发(六路各自写分片再合并),脚本会自动用这个方式:
|
|
python refresh.py --split 6
|
|
"""
|
|
import argparse
|
|
import glob
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
|
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
ROOT = os.path.dirname(HERE)
|
|
PY = sys.executable
|
|
|
|
|
|
def run(args, quiet=False):
|
|
cmd = [PY, os.path.join(HERE, "fetch_official.py")] + args
|
|
print("$", " ".join(args), flush=True)
|
|
p = subprocess.run(cmd, cwd=HERE)
|
|
if p.returncode != 0:
|
|
print("!! 非零退出:", p.returncode, " ".join(args), flush=True)
|
|
return p.returncode
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--workers", type=int, default=16)
|
|
ap.add_argument("--split", type=int, default=0, help="分几路并发扫,0=单路")
|
|
ap.add_argument("--skip-probe", action="store_true")
|
|
ap.add_argument("--days", type=int, default=45)
|
|
ap.add_argument("--limit", type=int, default=0, help="只处理前 N 所,用于自检")
|
|
ap.add_argument("--deep", action="store_true", help="先从已存正文反推缺入口院校的栏目页")
|
|
ap.add_argument("--no-attach", action="store_true", help="跳过附件抓取")
|
|
ap.add_argument("--sub", type=int, default=4, help="每校再跟进几个子栏目页,0=只看入口首屏")
|
|
a = ap.parse_args()
|
|
|
|
t0 = time.time()
|
|
print("=== 复查开始 %s ===" % time.strftime("%Y-%m-%d %H:%M:%S"))
|
|
|
|
run(["--derive"])
|
|
if a.deep:
|
|
# 有公告却没探到入口的院校,先从已存正文里反推栏目页,再统一探测。
|
|
cmd = [PY, os.path.join(HERE, "derive_from_body.py"), "--apply"]
|
|
print("$", "derive_from_body.py --apply", flush=True)
|
|
subprocess.run(cmd, cwd=HERE)
|
|
if not a.skip_probe:
|
|
run(["--probe", "--workers", str(a.workers)])
|
|
|
|
os.makedirs(os.path.join(HERE, "抓取分片"), exist_ok=True)
|
|
if a.split and a.split > 1:
|
|
procs = []
|
|
for i in range(a.split):
|
|
cmd = [PY, os.path.join(HERE, "fetch_official.py"), "--sweep",
|
|
"--shard", "%d/%d" % (i, a.split), "--workers", str(max(4, a.workers // a.split)),
|
|
"--out", os.path.join("抓取分片", "auto_%d.jsonl" % i)]
|
|
if a.skip_probe:
|
|
cmd.append("--only-empty")
|
|
cmd += ["--sub", str(a.sub)]
|
|
if a.limit:
|
|
cmd += ["--limit", str(a.limit)]
|
|
procs.append((subprocess.Popen(cmd, cwd=HERE), i))
|
|
for p, i in procs:
|
|
p.wait()
|
|
shards = sorted(glob.glob(os.path.join(HERE, "抓取分片", "auto_*.jsonl")))
|
|
run(["--ingest"] + [os.path.relpath(s, HERE) for s in shards])
|
|
else:
|
|
run(["--sweep", "--workers", str(a.workers)])
|
|
|
|
run(["--consolidate"]) # 合并重复必须先于回填,否则重复会被带进派生表
|
|
run(["--backfill"])
|
|
if not a.no_attach:
|
|
run(["--attach", "--workers", str(a.workers)])
|
|
run(["--report", "--days", str(a.days)])
|
|
att = os.path.join(ROOT, "公告抽取_2027附件字段.csv")
|
|
if os.path.exists(att):
|
|
cmd = [PY, os.path.join(HERE, "merge_att_fields.py")]
|
|
print("$", "merge_att_fields.py", flush=True)
|
|
subprocess.run(cmd, cwd=HERE)
|
|
for extra in ("reconcile_plan27.py", "report_coverage.py"):
|
|
ep = os.path.join(HERE, extra)
|
|
if os.path.exists(ep):
|
|
print("$", extra, flush=True)
|
|
subprocess.run([PY, ep], cwd=HERE)
|
|
cmd = [PY, os.path.join(HERE, "verify_db.py")]
|
|
print("$", "verify_db.py", flush=True)
|
|
subprocess.run(cmd, cwd=HERE)
|
|
print("=== 完成,用时 %.0f 秒 ===" % (time.time() - t0))
|
|
print("清单见 择校/最新通知.md 与 最新通知.csv;库里 notices_all 为最新。")
|
|
print("官方 0856 层需单独刷新时跑 integrate_chsi.py → align_chsi.py → export_api.py"
|
|
"(存档 JSON 有新增才需要)")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|