175 lines
7.0 KiB
Python
175 lines
7.0 KiB
Python
# -*- coding: utf-8 -*-
|
||
"""把复核补抓批次里混进正文的附件文本拆出来,附件改走 notice_files + attach。
|
||
|
||
背景:build_backfill_shard.py 早期版本把 hit_bodies.jsonl 的附件文本以
|
||
「附件《名字》: 正文」的形式拼在页面正文后面一起入库,15 条共 44.5 万字,
|
||
最长的华东政法大学一条被顶到 80000 字上限。派生表按正文抽字段时,
|
||
这些数字会被算成「公告正文口径」,与附件口径混在一起没法回溯。
|
||
|
||
本脚本按 prepare_miss_ingest.py 定的口径重做:
|
||
- body 只留容器抽取的页面正文,重算 body_len,并按入库同一份 mask_ids 脱敏;
|
||
- 附件原件按 fetch_official.one_download 落盘进 官方资料/最新通知2027/,
|
||
路径写进 notices_all.attach(JSON 数组)并登记 notice_files;
|
||
- 附件文本不再进 body,抽取由读原件的链路负责。
|
||
|
||
默认干跑,只报数不写库;--write 才落库,且落库前先复制一份数据库。
|
||
python 数据库/split_attach_from_review.py
|
||
python 数据库/split_attach_from_review.py --write
|
||
"""
|
||
import concurrent.futures as cf
|
||
import io
|
||
import json
|
||
import os
|
||
import re
|
||
import shutil
|
||
import sqlite3
|
||
import sys
|
||
import time
|
||
|
||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||
ROOT = os.path.dirname(HERE)
|
||
sys.path.insert(0, HERE)
|
||
|
||
import requests
|
||
|
||
import fetch_official as F
|
||
|
||
BODIES = os.path.join(ROOT, "复核工具", "hit_bodies.jsonl")
|
||
DB = os.path.join(HERE, "zexiao_2027.db")
|
||
LOG = os.path.join(HERE, "_split_attach.log")
|
||
DAY = "2026-09-26" # 复核补抓批次的入库日,只重做这一批
|
||
WS = re.compile(r"\s+")
|
||
ATT_MARK = re.compile(r"附件《")
|
||
BODY_MAX = 80000
|
||
WORKERS = int(os.environ.get("SPLIT_WORKERS", "8"))
|
||
|
||
|
||
def log(msg):
|
||
line = "%s %s" % (time.strftime("%H:%M:%S"), msg)
|
||
with io.open(LOG, "a", encoding="utf-8") as fh:
|
||
fh.write(line + "\n")
|
||
try:
|
||
print(line.encode("gbk", "replace").decode("gbk"), flush=True)
|
||
except Exception:
|
||
pass
|
||
|
||
|
||
def load_bodies():
|
||
"""按去锚点 URL 索引复核补抓的容器正文与附件清单。"""
|
||
out = {}
|
||
with io.open(BODIES, encoding="utf-8") as fh:
|
||
for line in fh:
|
||
line = line.strip()
|
||
if not line:
|
||
continue
|
||
rec = json.loads(line)
|
||
if rec.get("ok"):
|
||
out[rec["url"].split("#")[0]] = rec
|
||
return out
|
||
|
||
|
||
def main():
|
||
write = "--write" in sys.argv
|
||
try:
|
||
sys.stdout.reconfigure(encoding="utf-8")
|
||
except Exception:
|
||
pass
|
||
bodies = load_bodies()
|
||
con = sqlite3.connect(DB, timeout=60)
|
||
con.execute("PRAGMA busy_timeout=60000")
|
||
rows = list(con.execute(
|
||
"SELECT notice_key, school, title, url, body, body_len, attach "
|
||
"FROM notices_all WHERE source='official_web' AND found_at LIKE ?", (DAY + "%",)))
|
||
log("批次条目 %d 条,正文缓存 %d 条,模式 %s" % (len(rows), len(bodies), "写库" if write else "干跑"))
|
||
|
||
jobs, plans = [], []
|
||
no_cache = []
|
||
for key, school, title, url, body, blen, att in rows:
|
||
lu = (url or "").split("#")[0]
|
||
rec = bodies.get(lu)
|
||
if not rec:
|
||
no_cache.append((school, title, blen))
|
||
continue
|
||
page = WS.sub(" ", rec.get("body") or "").strip()
|
||
page = F.mask_ids(page)[:BODY_MAX] if page else None
|
||
atts = [a for a in (rec.get("atts") or []) if (a.get("url") or "").startswith("http")]
|
||
old = body or ""
|
||
tail = len(old) - (old.find("附件《") if ATT_MARK.search(old) else len(old))
|
||
plans.append({"key": key, "school": school, "title": title, "url": lu, "page": page,
|
||
"old_len": blen or 0, "att_extra": max(0, tail), "atts": atts,
|
||
"had_att": bool(att), "mixed": bool(ATT_MARK.search(old))})
|
||
for a in atts:
|
||
jobs.append((school, key, a["url"].split("#")[0], a.get("name") or title))
|
||
|
||
log("可重算 %d 条(其中正文混了附件的 %d 条);无补抓缓存 %d 条;待下载附件 %d 个"
|
||
% (len(plans), sum(1 for p in plans if p["mixed"]), len(no_cache), len(jobs)))
|
||
log("混入的附件段合计 %d 字" % sum(p["att_extra"] for p in plans if p["mixed"]))
|
||
if no_cache:
|
||
for s, t, b in no_cache[:10]:
|
||
log(" 无缓存: %s | len=%s | %s" % (s, b, (t or "")[:34]))
|
||
|
||
sess = requests.Session()
|
||
sess.headers.update(F.UA)
|
||
dl = {}
|
||
items = list(dict.fromkeys(jobs))
|
||
if write and items:
|
||
with cf.ThreadPoolExecutor(max_workers=WORKERS) as ex:
|
||
for r in ex.map(lambda it: F.one_download(sess, F.ARCHIVE, it), items):
|
||
dl.setdefault(r[0], []).append(r)
|
||
ok = sum(1 for v in dl.values() for r in v if r[7] == "ok")
|
||
log("附件落盘成功 %d / 登记 %d 条记录(失败 %d)"
|
||
% (ok, sum(len(v) for v in dl.values()), sum(len(v) for v in dl.values()) - ok))
|
||
|
||
n_body = n_att = n_fail_att = 0
|
||
if not write:
|
||
for p in plans:
|
||
n_body += 1 if p["mixed"] else 0
|
||
n_att += 1 if (p["atts"] and not p["had_att"]) else 0
|
||
log("干跑:将改正文 %d 条,待下载附件 %d 个,待回填 attach %d 条(未写库、未下载)"
|
||
% (n_body, len(items), n_att))
|
||
con.close()
|
||
return
|
||
|
||
bak = DB + ".bak_" + time.strftime("%H%M%S")
|
||
log("写库前备份 -> %s" % os.path.basename(bak))
|
||
src = sqlite3.connect(DB)
|
||
dst = sqlite3.connect(bak)
|
||
src.backup(dst)
|
||
dst.close()
|
||
src.close()
|
||
|
||
today = time.strftime("%Y-%m-%d")
|
||
for p in plans:
|
||
if p["mixed"]:
|
||
# 官网正文在交付 csv 里没有副本,只改确认混了附件的这 15 条,其余 body 原样不动
|
||
con.execute("UPDATE notices_all SET body=?, body_len=? WHERE notice_key=?",
|
||
(p["page"], len(p["page"] or ""), p["key"]))
|
||
n_body += 1
|
||
paths, fs = [], dl.get(p["key"], [])
|
||
for f in fs:
|
||
con.execute("INSERT OR IGNORE INTO notice_files VALUES (?,?,?,?,?,?,?,?,?,?)", f)
|
||
if f[7] == "ok":
|
||
paths.append(f[4])
|
||
else:
|
||
n_fail_att += 1
|
||
if paths and not p["had_att"]:
|
||
con.execute("UPDATE notices_all SET attach=? WHERE notice_key=?",
|
||
(json.dumps(paths, ensure_ascii=False), p["key"]))
|
||
n_att += 1
|
||
elif paths and p["had_att"]:
|
||
old = json.loads(p["had_att"]) if str(p["had_att"]).startswith("[") else []
|
||
merged = list(dict.fromkeys(old + paths))
|
||
con.execute("UPDATE notices_all SET attach=? WHERE notice_key=?",
|
||
(json.dumps(merged, ensure_ascii=False), p["key"]))
|
||
n_att += 1
|
||
con.execute("INSERT INTO sweep_log VALUES (NULL,?,?,?,?,?,?,?)",
|
||
("复核批次正文拆分", None, len(plans), n_body, n_att, n_fail_att,
|
||
time.strftime("%Y-%m-%d %H:%M")))
|
||
con.commit()
|
||
log("完成:改正文 %d 条,回填 attach %d 条,附件失败 %d 个" % (n_body, n_att, n_fail_att))
|
||
con.close()
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|