补齐页面线表的来源URL并留缺项清单
广外那95行是从agent落盘的HTML抽的,采集清单没记地址。 采集索引查不到时退回按校名+标题在 notices_all 回查,命中同篇公告 (2026-03-13 关于公布我校2026年硕士研究生招生复试分数线和复试名单的通知)后回填。 回查脚本会把补不上的文件名如实列出来而不是静默留空, 交付表里每个数字都要能当场回溯到源文件。
This commit is contained in:
1 parent
7a26120de4
commit
a319e90565
3 files changed
+181
No files matched your search
@@ -0,0 +1,67 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""给 official_score_lines 里缺来源的行回填 URL。
|
||||
|
||||
页面抽出的线(广东外语外贸大学那 95 行)当初没带 url,交付表里每个数字都要能
|
||||
当场回溯到源文件,所以按采集清单回填;回不到的一律列出来而不是留空装没事。
|
||||
"""
|
||||
import csv
|
||||
import os
|
||||
import re
|
||||
import sqlite3
|
||||
import time
|
||||
|
||||
ZX = r'D:/择校/数据库/zexiao_2027.db'
|
||||
DIR = r'D:/择校/运行暂存/official_roster_2025_2026'
|
||||
NAME_KEYS = ('local_path', 'local', 'file', 'saved_as', 'path')
|
||||
URL_KEYS = ('url', 'page_url', 'link', 'source_url')
|
||||
|
||||
|
||||
def load_index():
|
||||
idx = {}
|
||||
for fn in ('pages_manifest.csv', 'links.csv', 'links_p2.csv', 'download_manifest.csv'):
|
||||
p = os.path.join(DIR, fn)
|
||||
if not os.path.exists(p):
|
||||
continue
|
||||
for r in csv.DictReader(open(p, encoding='utf-8-sig')):
|
||||
name = next((str(r[k]) for k in NAME_KEYS if r.get(k)), '')
|
||||
url = next((str(r[k]) for k in URL_KEYS if r.get(k)), '')
|
||||
if not name or not url:
|
||||
continue
|
||||
b = os.path.basename(re.sub(r'[\\/]+', '/', name))
|
||||
idx.setdefault(b, url)
|
||||
idx.setdefault(os.path.splitext(b)[0] + '.pdf', url)
|
||||
return idx
|
||||
|
||||
|
||||
def main():
|
||||
z = sqlite3.connect(ZX, timeout=340)
|
||||
z.execute('PRAGMA busy_timeout=340000')
|
||||
idx = load_index()
|
||||
print('URL 索引=%d 条' % len(idx))
|
||||
miss = [r[0] for r in z.execute(
|
||||
"select distinct local_file from official_score_lines where trim(source_url)=''")]
|
||||
n = 0
|
||||
hit = []
|
||||
for f in miss:
|
||||
u = idx.get(f) or idx.get(os.path.splitext(f)[0] + '.pdf') or idx.get(os.path.splitext(f)[0])
|
||||
if u:
|
||||
n += z.execute("update official_score_lines set source_url=? where local_file=? "
|
||||
"and trim(source_url)=''", (u, f)).rowcount
|
||||
hit.append(f)
|
||||
if n:
|
||||
z.execute('INSERT INTO changelog VALUES (?,?,?,?,?,?)',
|
||||
(time.strftime('%Y-%m-%d %H:%M:%S'), 'line_source_backfill_20261101',
|
||||
'official_score_lines', 'update', n, '页面线表按采集清单回填来源URL'))
|
||||
z.commit()
|
||||
print('回填=%d 涉及文件=%s' % (n, hit))
|
||||
left = [r[0] for r in z.execute(
|
||||
"select distinct local_file from official_score_lines where trim(source_url)=''")]
|
||||
print('仍无来源的文件=%s' % left)
|
||||
for t in ('official_score_lines', 'official_roster_major_stat', 'web_accommodation_policy'):
|
||||
print(' %s 无来源行=%d' % (t, z.execute(
|
||||
"select count(*) from %s where source_url is null or trim(source_url)=''" % t).fetchone()[0]))
|
||||
z.close()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,49 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""用主库公告注册表给缺来源的线行找 URL。
|
||||
|
||||
广外那 95 行是从 agent 落盘的 HTML 抽的,采集清单里没记它的地址,
|
||||
但 notices_all 里大概率登记过同一篇公告,按校名+标题关键词回查比留空好。
|
||||
"""
|
||||
import re
|
||||
import sqlite3
|
||||
import time
|
||||
|
||||
ZX = r'D:/择校/数据库/zexiao_2027.db'
|
||||
|
||||
|
||||
def main():
|
||||
z = sqlite3.connect(ZX, timeout=340)
|
||||
z.execute('PRAGMA busy_timeout=340000')
|
||||
for f in [r[0] for r in z.execute(
|
||||
"select distinct local_file from official_score_lines where trim(source_url)=''")]:
|
||||
school = re.match(r'([一-龥]+?)(?:_|大学|学院)', f)
|
||||
sch = school.group(1) if school else f.split('_')[0]
|
||||
if '大学' not in sch and '学院' not in sch:
|
||||
sch = f.replace('_2026_page.html', '').replace('.html', '')
|
||||
cands = z.execute("""select url,title,pub_date from notices_all
|
||||
where school=? and (title like '%复试线%' or title like '%分数线%' or body like '%总分复试线%')
|
||||
order by pub_date desc limit 5""", (sch,)).fetchall()
|
||||
if not cands:
|
||||
cands = z.execute("""select url,title,pub_date from notices_all
|
||||
where title like ? and (title like '%复试%' or title like '%分数%') limit 5""",
|
||||
('%%%s%%' % sch,)).fetchall()
|
||||
print('%-34s 查=%s 命中=%d' % (f, sch, len(cands)))
|
||||
for u, t, p in cands[:3]:
|
||||
print(' %s | %s' % ((p or '')[:10], (t or '')[:70]))
|
||||
if cands:
|
||||
url, title, pub = cands[0]
|
||||
n = z.execute("update official_score_lines set source_url=?, note=note||';URL由notices_all按校名+标题回查' "
|
||||
"where local_file=? and trim(source_url)=''", (url, f)).rowcount
|
||||
z.execute('INSERT INTO changelog VALUES (?,?,?,?,?,?)',
|
||||
(time.strftime('%Y-%m-%d %H:%M:%S'), 'line_source_backfill_20261101',
|
||||
'official_score_lines', 'update', n,
|
||||
'广外页面线从 notices_all 回查URL:%s' % (title or '')[:60]))
|
||||
z.commit()
|
||||
print(' 回填=%d 行' % n)
|
||||
print('仍无来源=', z.execute(
|
||||
"select count(*) from official_score_lines where trim(source_url)=''").fetchone()[0])
|
||||
z.close()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,65 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""为 0856 缺口院校定位库内已登记的 2027 推免公示入口。
|
||||
|
||||
只读查询。产出是「定向目标清单」而不是数据本身:
|
||||
库里 notices_all/notice_files 已经登记过大量官方公告,
|
||||
先按标题把推免公示筛出来,比让采集 agent 从外部重新撒网准得多。
|
||||
"""
|
||||
import csv
|
||||
import os
|
||||
import re
|
||||
import sqlite3
|
||||
|
||||
ZX = r'D:/择校/数据库/zexiao_2027.db'
|
||||
GAP = r'D:/择校/审计报告/0856_2027定向缺口清单.csv'
|
||||
OUT = r'D:/择校/运行暂存/agent_ingest_20261101/tuimian_targets_0856.csv'
|
||||
|
||||
TM = re.compile(r'推免|推荐免试|免试.*研究生|免推|接收.*推免')
|
||||
Y27 = re.compile(r'2027|2027年')
|
||||
NEG = re.compile(r'博士|硕博连读|强军|退役|支教')
|
||||
|
||||
|
||||
def main():
|
||||
gap = list(csv.DictReader(open(GAP, encoding='utf-8-sig')))
|
||||
need = {}
|
||||
for r in gap:
|
||||
if r['缺推免接收数'] == '是':
|
||||
need.setdefault(r['院校'], set()).add(r['专业代码'])
|
||||
print('待补院校=%d' % len(need))
|
||||
z = sqlite3.connect(ZX, timeout=200)
|
||||
z.execute('PRAGMA busy_timeout=200000')
|
||||
names = list(need)
|
||||
hits = []
|
||||
ph = ','.join('?' * len(names))
|
||||
for sch, title, url, body, pub, attach, nf in z.execute(
|
||||
'select school,title,url,body,pub_date,attach,notice_key from notices_all '
|
||||
'where school in (%s)' % ph, names):
|
||||
t = title or ''
|
||||
if not TM.search(t) or not Y27.search(t):
|
||||
continue
|
||||
kind = '推免公示'
|
||||
ev = 'title'
|
||||
if NEG.search(t) and not re.search(r'硕士', t):
|
||||
kind = '专项/博士推免'
|
||||
hits.append(dict(school=sch, title=t[:110], url=url or '', pub_date=pub or '',
|
||||
has_attach='是' if (attach or '').strip() else '',
|
||||
notice_key=nf, kind=kind, matched_by=ev,
|
||||
codes_needed=';'.join(sorted(need[sch])[:4]),
|
||||
body_len=len(body or '')))
|
||||
print('命中公告=%d 涉及校=%d' % (len(hits), len({h['school'] for h in hits})))
|
||||
with open(OUT, 'w', newline='', encoding='utf-8-sig') as f:
|
||||
w = csv.DictWriter(f, fieldnames=['school', 'title', 'url', 'pub_date', 'has_attach',
|
||||
'notice_key', 'kind', 'body_len'])
|
||||
w.writeheader()
|
||||
for h in hits:
|
||||
w.writerow({k: h[k] for k in w.fieldnames})
|
||||
miss = sorted(set(need) - {h['school'] for h in hits})
|
||||
print('库内无登记的校=%d,例:%s' % (len(miss), miss[:8]))
|
||||
print('写入', OUT, os.path.getsize(OUT), 'B')
|
||||
for h in hits[:5]:
|
||||
print(' %-14s %s' % (h['school'][:14], h['title'][:64]))
|
||||
z.close()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in new issue
Block a user