diff --git a/脚本/来源回查/backfill_line_source.py b/脚本/来源回查/backfill_line_source.py new file mode 100644 index 000000000..e098aaac5 --- /dev/null +++ b/脚本/来源回查/backfill_line_source.py @@ -0,0 +1,67 @@ +# -*- coding: utf-8 -*- +"""给 official_score_lines 里缺来源的行回填 URL。 + +页面抽出的线(广东外语外贸大学那 95 行)当初没带 url,交付表里每个数字都要能 +当场回溯到源文件,所以按采集清单回填;回不到的一律列出来而不是留空装没事。 +""" +import csv +import os +import re +import sqlite3 +import time + +ZX = r'D:/择校/数据库/zexiao_2027.db' +DIR = r'D:/择校/运行暂存/official_roster_2025_2026' +NAME_KEYS = ('local_path', 'local', 'file', 'saved_as', 'path') +URL_KEYS = ('url', 'page_url', 'link', 'source_url') + + +def load_index(): + idx = {} + for fn in ('pages_manifest.csv', 'links.csv', 'links_p2.csv', 'download_manifest.csv'): + p = os.path.join(DIR, fn) + if not os.path.exists(p): + continue + for r in csv.DictReader(open(p, encoding='utf-8-sig')): + name = next((str(r[k]) for k in NAME_KEYS if r.get(k)), '') + url = next((str(r[k]) for k in URL_KEYS if r.get(k)), '') + if not name or not url: + continue + b = os.path.basename(re.sub(r'[\\/]+', '/', name)) + idx.setdefault(b, url) + idx.setdefault(os.path.splitext(b)[0] + '.pdf', url) + return idx + + +def main(): + z = sqlite3.connect(ZX, timeout=340) + z.execute('PRAGMA busy_timeout=340000') + idx = load_index() + print('URL 索引=%d 条' % len(idx)) + miss = [r[0] for r in z.execute( + "select distinct local_file from official_score_lines where trim(source_url)=''")] + n = 0 + hit = [] + for f in miss: + u = idx.get(f) or idx.get(os.path.splitext(f)[0] + '.pdf') or idx.get(os.path.splitext(f)[0]) + if u: + n += z.execute("update official_score_lines set source_url=? where local_file=? " + "and trim(source_url)=''", (u, f)).rowcount + hit.append(f) + if n: + z.execute('INSERT INTO changelog VALUES (?,?,?,?,?,?)', + (time.strftime('%Y-%m-%d %H:%M:%S'), 'line_source_backfill_20261101', + 'official_score_lines', 'update', n, '页面线表按采集清单回填来源URL')) + z.commit() + print('回填=%d 涉及文件=%s' % (n, hit)) + left = [r[0] for r in z.execute( + "select distinct local_file from official_score_lines where trim(source_url)=''")] + print('仍无来源的文件=%s' % left) + for t in ('official_score_lines', 'official_roster_major_stat', 'web_accommodation_policy'): + print(' %s 无来源行=%d' % (t, z.execute( + "select count(*) from %s where source_url is null or trim(source_url)=''" % t).fetchone()[0])) + z.close() + + +if __name__ == '__main__': + main() diff --git a/脚本/来源回查/backfill_line_source2.py b/脚本/来源回查/backfill_line_source2.py new file mode 100644 index 000000000..111c88bc2 --- /dev/null +++ b/脚本/来源回查/backfill_line_source2.py @@ -0,0 +1,49 @@ +# -*- coding: utf-8 -*- +"""用主库公告注册表给缺来源的线行找 URL。 + +广外那 95 行是从 agent 落盘的 HTML 抽的,采集清单里没记它的地址, +但 notices_all 里大概率登记过同一篇公告,按校名+标题关键词回查比留空好。 +""" +import re +import sqlite3 +import time + +ZX = r'D:/择校/数据库/zexiao_2027.db' + + +def main(): + z = sqlite3.connect(ZX, timeout=340) + z.execute('PRAGMA busy_timeout=340000') + for f in [r[0] for r in z.execute( + "select distinct local_file from official_score_lines where trim(source_url)=''")]: + school = re.match(r'([一-龥]+?)(?:_|大学|学院)', f) + sch = school.group(1) if school else f.split('_')[0] + if '大学' not in sch and '学院' not in sch: + sch = f.replace('_2026_page.html', '').replace('.html', '') + cands = z.execute("""select url,title,pub_date from notices_all + where school=? and (title like '%复试线%' or title like '%分数线%' or body like '%总分复试线%') + order by pub_date desc limit 5""", (sch,)).fetchall() + if not cands: + cands = z.execute("""select url,title,pub_date from notices_all + where title like ? and (title like '%复试%' or title like '%分数%') limit 5""", + ('%%%s%%' % sch,)).fetchall() + print('%-34s 查=%s 命中=%d' % (f, sch, len(cands))) + for u, t, p in cands[:3]: + print(' %s | %s' % ((p or '')[:10], (t or '')[:70])) + if cands: + url, title, pub = cands[0] + n = z.execute("update official_score_lines set source_url=?, note=note||';URL由notices_all按校名+标题回查' " + "where local_file=? and trim(source_url)=''", (url, f)).rowcount + z.execute('INSERT INTO changelog VALUES (?,?,?,?,?,?)', + (time.strftime('%Y-%m-%d %H:%M:%S'), 'line_source_backfill_20261101', + 'official_score_lines', 'update', n, + '广外页面线从 notices_all 回查URL:%s' % (title or '')[:60])) + z.commit() + print(' 回填=%d 行' % n) + print('仍无来源=', z.execute( + "select count(*) from official_score_lines where trim(source_url)=''").fetchone()[0]) + z.close() + + +if __name__ == '__main__': + main() diff --git a/脚本/来源回查/find_tuimian_targets.py b/脚本/来源回查/find_tuimian_targets.py new file mode 100644 index 000000000..636b8d043 --- /dev/null +++ b/脚本/来源回查/find_tuimian_targets.py @@ -0,0 +1,65 @@ +# -*- coding: utf-8 -*- +"""为 0856 缺口院校定位库内已登记的 2027 推免公示入口。 + +只读查询。产出是「定向目标清单」而不是数据本身: +库里 notices_all/notice_files 已经登记过大量官方公告, +先按标题把推免公示筛出来,比让采集 agent 从外部重新撒网准得多。 +""" +import csv +import os +import re +import sqlite3 + +ZX = r'D:/择校/数据库/zexiao_2027.db' +GAP = r'D:/择校/审计报告/0856_2027定向缺口清单.csv' +OUT = r'D:/择校/运行暂存/agent_ingest_20261101/tuimian_targets_0856.csv' + +TM = re.compile(r'推免|推荐免试|免试.*研究生|免推|接收.*推免') +Y27 = re.compile(r'2027|2027年') +NEG = re.compile(r'博士|硕博连读|强军|退役|支教') + + +def main(): + gap = list(csv.DictReader(open(GAP, encoding='utf-8-sig'))) + need = {} + for r in gap: + if r['缺推免接收数'] == '是': + need.setdefault(r['院校'], set()).add(r['专业代码']) + print('待补院校=%d' % len(need)) + z = sqlite3.connect(ZX, timeout=200) + z.execute('PRAGMA busy_timeout=200000') + names = list(need) + hits = [] + ph = ','.join('?' * len(names)) + for sch, title, url, body, pub, attach, nf in z.execute( + 'select school,title,url,body,pub_date,attach,notice_key from notices_all ' + 'where school in (%s)' % ph, names): + t = title or '' + if not TM.search(t) or not Y27.search(t): + continue + kind = '推免公示' + ev = 'title' + if NEG.search(t) and not re.search(r'硕士', t): + kind = '专项/博士推免' + hits.append(dict(school=sch, title=t[:110], url=url or '', pub_date=pub or '', + has_attach='是' if (attach or '').strip() else '', + notice_key=nf, kind=kind, matched_by=ev, + codes_needed=';'.join(sorted(need[sch])[:4]), + body_len=len(body or ''))) + print('命中公告=%d 涉及校=%d' % (len(hits), len({h['school'] for h in hits}))) + with open(OUT, 'w', newline='', encoding='utf-8-sig') as f: + w = csv.DictWriter(f, fieldnames=['school', 'title', 'url', 'pub_date', 'has_attach', + 'notice_key', 'kind', 'body_len']) + w.writeheader() + for h in hits: + w.writerow({k: h[k] for k in w.fieldnames}) + miss = sorted(set(need) - {h['school'] for h in hits}) + print('库内无登记的校=%d,例:%s' % (len(miss), miss[:8])) + print('写入', OUT, os.path.getsize(OUT), 'B') + for h in hits[:5]: + print(' %-14s %s' % (h['school'][:14], h['title'][:64])) + z.close() + + +if __name__ == '__main__': + main()