Files
yanxuan/数据库/split_attach_from_review.py

175 lines
7.0 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
# -*- coding: utf-8 -*-
"""把复核补抓批次里混进正文的附件文本拆出来,附件改走 notice_files + attach。
背景:build_backfill_shard.py 早期版本把 hit_bodies.jsonl 的附件文本以
「附件《名字》: 正文」的形式拼在页面正文后面一起入库,15 条共 44.5 万字,
最长的华东政法大学一条被顶到 80000 字上限。派生表按正文抽字段时,
这些数字会被算成「公告正文口径」,与附件口径混在一起没法回溯。
本脚本按 prepare_miss_ingest.py 定的口径重做:
- body 只留容器抽取的页面正文,重算 body_len,并按入库同一份 mask_ids 脱敏;
- 附件原件按 fetch_official.one_download 落盘进 官方资料/最新通知2027/,
路径写进 notices_all.attach(JSON 数组)并登记 notice_files;
- 附件文本不再进 body,抽取由读原件的链路负责。
默认干跑,只报数不写库;--write 才落库,且落库前先复制一份数据库。
python 数据库/split_attach_from_review.py
python 数据库/split_attach_from_review.py --write
"""
import concurrent.futures as cf
import io
import json
import os
import re
import shutil
import sqlite3
import sys
import time
HERE = os.path.dirname(os.path.abspath(__file__))
ROOT = os.path.dirname(HERE)
sys.path.insert(0, HERE)
import requests
import fetch_official as F
BODIES = os.path.join(ROOT, "复核工具", "hit_bodies.jsonl")
DB = os.path.join(HERE, "zexiao_2027.db")
LOG = os.path.join(HERE, "_split_attach.log")
DAY = "2026-09-26" # 复核补抓批次的入库日,只重做这一批
WS = re.compile(r"\s+")
ATT_MARK = re.compile(r"附件《")
BODY_MAX = 80000
WORKERS = int(os.environ.get("SPLIT_WORKERS", "8"))
def log(msg):
line = "%s %s" % (time.strftime("%H:%M:%S"), msg)
with io.open(LOG, "a", encoding="utf-8") as fh:
fh.write(line + "\n")
try:
print(line.encode("gbk", "replace").decode("gbk"), flush=True)
except Exception:
pass
def load_bodies():
"""按去锚点 URL 索引复核补抓的容器正文与附件清单。"""
out = {}
with io.open(BODIES, encoding="utf-8") as fh:
for line in fh:
line = line.strip()
if not line:
continue
rec = json.loads(line)
if rec.get("ok"):
out[rec["url"].split("#")[0]] = rec
return out
def main():
write = "--write" in sys.argv
try:
sys.stdout.reconfigure(encoding="utf-8")
except Exception:
pass
bodies = load_bodies()
con = sqlite3.connect(DB, timeout=60)
con.execute("PRAGMA busy_timeout=60000")
rows = list(con.execute(
"SELECT notice_key, school, title, url, body, body_len, attach "
"FROM notices_all WHERE source='official_web' AND found_at LIKE ?", (DAY + "%",)))
log("批次条目 %d 条,正文缓存 %d 条,模式 %s" % (len(rows), len(bodies), "写库" if write else "干跑"))
jobs, plans = [], []
no_cache = []
for key, school, title, url, body, blen, att in rows:
lu = (url or "").split("#")[0]
rec = bodies.get(lu)
if not rec:
no_cache.append((school, title, blen))
continue
page = WS.sub(" ", rec.get("body") or "").strip()
page = F.mask_ids(page)[:BODY_MAX] if page else None
atts = [a for a in (rec.get("atts") or []) if (a.get("url") or "").startswith("http")]
old = body or ""
tail = len(old) - (old.find("附件《") if ATT_MARK.search(old) else len(old))
plans.append({"key": key, "school": school, "title": title, "url": lu, "page": page,
"old_len": blen or 0, "att_extra": max(0, tail), "atts": atts,
"had_att": bool(att), "mixed": bool(ATT_MARK.search(old))})
for a in atts:
jobs.append((school, key, a["url"].split("#")[0], a.get("name") or title))
log("可重算 %d 条(其中正文混了附件的 %d 条);无补抓缓存 %d 条;待下载附件 %d 个"
% (len(plans), sum(1 for p in plans if p["mixed"]), len(no_cache), len(jobs)))
log("混入的附件段合计 %d 字" % sum(p["att_extra"] for p in plans if p["mixed"]))
if no_cache:
for s, t, b in no_cache[:10]:
log(" 无缓存: %s | len=%s | %s" % (s, b, (t or "")[:34]))
sess = requests.Session()
sess.headers.update(F.UA)
dl = {}
items = list(dict.fromkeys(jobs))
if write and items:
with cf.ThreadPoolExecutor(max_workers=WORKERS) as ex:
for r in ex.map(lambda it: F.one_download(sess, F.ARCHIVE, it), items):
dl.setdefault(r[0], []).append(r)
ok = sum(1 for v in dl.values() for r in v if r[7] == "ok")
log("附件落盘成功 %d / 登记 %d 条记录(失败 %d)"
% (ok, sum(len(v) for v in dl.values()), sum(len(v) for v in dl.values()) - ok))
n_body = n_att = n_fail_att = 0
if not write:
for p in plans:
n_body += 1 if p["mixed"] else 0
n_att += 1 if (p["atts"] and not p["had_att"]) else 0
log("干跑:将改正文 %d 条,待下载附件 %d 个,待回填 attach %d 条(未写库、未下载)"
% (n_body, len(items), n_att))
con.close()
return
bak = DB + ".bak_" + time.strftime("%H%M%S")
log("写库前备份 -> %s" % os.path.basename(bak))
src = sqlite3.connect(DB)
dst = sqlite3.connect(bak)
src.backup(dst)
dst.close()
src.close()
today = time.strftime("%Y-%m-%d")
for p in plans:
if p["mixed"]:
# 官网正文在交付 csv 里没有副本,只改确认混了附件的这 15 条,其余 body 原样不动
con.execute("UPDATE notices_all SET body=?, body_len=? WHERE notice_key=?",
(p["page"], len(p["page"] or ""), p["key"]))
n_body += 1
paths, fs = [], dl.get(p["key"], [])
for f in fs:
con.execute("INSERT OR IGNORE INTO notice_files VALUES (?,?,?,?,?,?,?,?,?,?)", f)
if f[7] == "ok":
paths.append(f[4])
else:
n_fail_att += 1
if paths and not p["had_att"]:
con.execute("UPDATE notices_all SET attach=? WHERE notice_key=?",
(json.dumps(paths, ensure_ascii=False), p["key"]))
n_att += 1
elif paths and p["had_att"]:
old = json.loads(p["had_att"]) if str(p["had_att"]).startswith("[") else []
merged = list(dict.fromkeys(old + paths))
con.execute("UPDATE notices_all SET attach=? WHERE notice_key=?",
(json.dumps(merged, ensure_ascii=False), p["key"]))
n_att += 1
con.execute("INSERT INTO sweep_log VALUES (NULL,?,?,?,?,?,?,?)",
("复核批次正文拆分", None, len(plans), n_body, n_att, n_fail_att,
time.strftime("%Y-%m-%d %H:%M")))
con.commit()
log("完成:改正文 %d 条,回填 attach %d 条,附件失败 %d 个" % (n_body, n_att, n_fail_att))
con.close()
if __name__ == "__main__":
main()