补齐页面线表的来源URL并留缺项清单

广外那95行是从agent落盘的HTML抽的,采集清单没记地址。
采集索引查不到时退回按校名+标题在 notices_all 回查,命中同篇公告
(2026-03-13 关于公布我校2026年硕士研究生招生复试分数线和复试名单的通知)后回填。
回查脚本会把补不上的文件名如实列出来而不是静默留空,
交付表里每个数字都要能当场回溯到源文件。
This commit is contained in:
Your Name committed 2026-11-01 01:57:15 +08:00
1 parent 7a26120de4
commit a319e90565
3 files changed
+181

No files matched your search

@@ -0,0 +1,67 @@
# -*- coding: utf-8 -*-
"""给 official_score_lines 里缺来源的行回填 URL。
页面抽出的线(广东外语外贸大学那 95 行)当初没带 url,交付表里每个数字都要能
当场回溯到源文件,所以按采集清单回填;回不到的一律列出来而不是留空装没事。
"""
import csv
import os
import re
import sqlite3
import time
ZX = r'D:/择校/数据库/zexiao_2027.db'
DIR = r'D:/择校/运行暂存/official_roster_2025_2026'
NAME_KEYS = ('local_path', 'local', 'file', 'saved_as', 'path')
URL_KEYS = ('url', 'page_url', 'link', 'source_url')
def load_index():
idx = {}
for fn in ('pages_manifest.csv', 'links.csv', 'links_p2.csv', 'download_manifest.csv'):
p = os.path.join(DIR, fn)
if not os.path.exists(p):
continue
for r in csv.DictReader(open(p, encoding='utf-8-sig')):
name = next((str(r[k]) for k in NAME_KEYS if r.get(k)), '')
url = next((str(r[k]) for k in URL_KEYS if r.get(k)), '')
if not name or not url:
continue
b = os.path.basename(re.sub(r'[\\/]+', '/', name))
idx.setdefault(b, url)
idx.setdefault(os.path.splitext(b)[0] + '.pdf', url)
return idx
def main():
z = sqlite3.connect(ZX, timeout=340)
z.execute('PRAGMA busy_timeout=340000')
idx = load_index()
print('URL 索引=%d 条' % len(idx))
miss = [r[0] for r in z.execute(
"select distinct local_file from official_score_lines where trim(source_url)=''")]
n = 0
hit = []
for f in miss:
u = idx.get(f) or idx.get(os.path.splitext(f)[0] + '.pdf') or idx.get(os.path.splitext(f)[0])
if u:
n += z.execute("update official_score_lines set source_url=? where local_file=? "
"and trim(source_url)=''", (u, f)).rowcount
hit.append(f)
if n:
z.execute('INSERT INTO changelog VALUES (?,?,?,?,?,?)',
(time.strftime('%Y-%m-%d %H:%M:%S'), 'line_source_backfill_20261101',
'official_score_lines', 'update', n, '页面线表按采集清单回填来源URL'))
z.commit()
print('回填=%d 涉及文件=%s' % (n, hit))
left = [r[0] for r in z.execute(
"select distinct local_file from official_score_lines where trim(source_url)=''")]
print('仍无来源的文件=%s' % left)
for t in ('official_score_lines', 'official_roster_major_stat', 'web_accommodation_policy'):
print(' %s 无来源行=%d' % (t, z.execute(
"select count(*) from %s where source_url is null or trim(source_url)=''" % t).fetchone()[0]))
z.close()
if __name__ == '__main__':
main()
@@ -0,0 +1,49 @@
# -*- coding: utf-8 -*-
"""用主库公告注册表给缺来源的线行找 URL。
广外那 95 行是从 agent 落盘的 HTML 抽的,采集清单里没记它的地址,
但 notices_all 里大概率登记过同一篇公告,按校名+标题关键词回查比留空好。
"""
import re
import sqlite3
import time
ZX = r'D:/择校/数据库/zexiao_2027.db'
def main():
z = sqlite3.connect(ZX, timeout=340)
z.execute('PRAGMA busy_timeout=340000')
for f in [r[0] for r in z.execute(
"select distinct local_file from official_score_lines where trim(source_url)=''")]:
school = re.match(r'([一-龥]+?)(?:_|大学|学院)', f)
sch = school.group(1) if school else f.split('_')[0]
if '大学' not in sch and '学院' not in sch:
sch = f.replace('_2026_page.html', '').replace('.html', '')
cands = z.execute("""select url,title,pub_date from notices_all
where school=? and (title like '%复试线%' or title like '%分数线%' or body like '%总分复试线%')
order by pub_date desc limit 5""", (sch,)).fetchall()
if not cands:
cands = z.execute("""select url,title,pub_date from notices_all
where title like ? and (title like '%复试%' or title like '%分数%') limit 5""",
('%%%s%%' % sch,)).fetchall()
print('%-34s 查=%s 命中=%d' % (f, sch, len(cands)))
for u, t, p in cands[:3]:
print(' %s | %s' % ((p or '')[:10], (t or '')[:70]))
if cands:
url, title, pub = cands[0]
n = z.execute("update official_score_lines set source_url=?, note=note||';URL由notices_all按校名+标题回查' "
"where local_file=? and trim(source_url)=''", (url, f)).rowcount
z.execute('INSERT INTO changelog VALUES (?,?,?,?,?,?)',
(time.strftime('%Y-%m-%d %H:%M:%S'), 'line_source_backfill_20261101',
'official_score_lines', 'update', n,
'广外页面线从 notices_all 回查URL:%s' % (title or '')[:60]))
z.commit()
print(' 回填=%d 行' % n)
print('仍无来源=', z.execute(
"select count(*) from official_score_lines where trim(source_url)=''").fetchone()[0])
z.close()
if __name__ == '__main__':
main()
@@ -0,0 +1,65 @@
# -*- coding: utf-8 -*-
"""为 0856 缺口院校定位库内已登记的 2027 推免公示入口。
只读查询。产出是「定向目标清单」而不是数据本身:
库里 notices_all/notice_files 已经登记过大量官方公告,
先按标题把推免公示筛出来,比让采集 agent 从外部重新撒网准得多。
"""
import csv
import os
import re
import sqlite3
ZX = r'D:/择校/数据库/zexiao_2027.db'
GAP = r'D:/择校/审计报告/0856_2027定向缺口清单.csv'
OUT = r'D:/择校/运行暂存/agent_ingest_20261101/tuimian_targets_0856.csv'
TM = re.compile(r'推免|推荐免试|免试.*研究生|免推|接收.*推免')
Y27 = re.compile(r'2027|2027年')
NEG = re.compile(r'博士|硕博连读|强军|退役|支教')
def main():
gap = list(csv.DictReader(open(GAP, encoding='utf-8-sig')))
need = {}
for r in gap:
if r['缺推免接收数'] == '是':
need.setdefault(r['院校'], set()).add(r['专业代码'])
print('待补院校=%d' % len(need))
z = sqlite3.connect(ZX, timeout=200)
z.execute('PRAGMA busy_timeout=200000')
names = list(need)
hits = []
ph = ','.join('?' * len(names))
for sch, title, url, body, pub, attach, nf in z.execute(
'select school,title,url,body,pub_date,attach,notice_key from notices_all '
'where school in (%s)' % ph, names):
t = title or ''
if not TM.search(t) or not Y27.search(t):
continue
kind = '推免公示'
ev = 'title'
if NEG.search(t) and not re.search(r'硕士', t):
kind = '专项/博士推免'
hits.append(dict(school=sch, title=t[:110], url=url or '', pub_date=pub or '',
has_attach='是' if (attach or '').strip() else '',
notice_key=nf, kind=kind, matched_by=ev,
codes_needed=';'.join(sorted(need[sch])[:4]),
body_len=len(body or '')))
print('命中公告=%d 涉及校=%d' % (len(hits), len({h['school'] for h in hits})))
with open(OUT, 'w', newline='', encoding='utf-8-sig') as f:
w = csv.DictWriter(f, fieldnames=['school', 'title', 'url', 'pub_date', 'has_attach',
'notice_key', 'kind', 'body_len'])
w.writeheader()
for h in hits:
w.writerow({k: h[k] for k in w.fieldnames})
miss = sorted(set(need) - {h['school'] for h in hits})
print('库内无登记的校=%d,例:%s' % (len(miss), miss[:8]))
print('写入', OUT, os.path.getsize(OUT), 'B')
for h in hits[:5]:
print(' %-14s %s' % (h['school'][:14], h['title'][:64]))
z.close()
if __name__ == '__main__':
main()