diff --git a/脚本/名单解析/fix_basis.py b/脚本/名单解析/fix_basis.py new file mode 100644 index 000000000..dc853d39b --- /dev/null +++ b/脚本/名单解析/fix_basis.py @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- +"""补 basis 缺失并查清每条的出处,不是一刀切填 roster_derived。 + +y2027_tuimian_2027 靠 basis 区分「官方计数表」与「名单分组推导」, +空着等于让下游无法判断这个数能不能直接当接收人数用。 +""" +import sqlite3 +import time + +ZX = r'D:/择校/数据库/zexiao_2027.db' +z = sqlite3.connect(ZX, timeout=400) +z.execute('PRAGMA busy_timeout=400000') + +bad = z.execute("select local_file,source_url,count(*) from y2027_tuimian_2027 " + "where trim(basis)='' group by 1,2 order by 3 desc").fetchall() +print('basis 空涉及原件:') +for r in bad: + print(' %-34s %s 行=%d' % (str(r[0])[:34], str(r[1])[-28:], r[2])) + +# 名单分组型:原件是逐人公示(PDF 名含 nlq/md 且无计数列); +# 这里按解析器实际产出路径判:cand_batch(PDF 走 parse_official_roster_stat)只有名单分组一种产出 +pdfc = [r[0] for r in bad if (r[0] or '').endswith('.pdf')] +htmlc = [r[0] for r in bad if (r[0] or '').endswith('.html')] +n = 0 +for f in pdfc: + n += z.execute("update y2027_tuimian_2027 set basis='roster_derived', " + "note=note||';名单分组行数(补记 basis)' " + "where trim(basis)='' and local_file=? and note like '%0856定向批%'", (f,)).rowcount +print('PDF 原件补 basis=%d 行' % n) +n2 = 0 +for f in htmlc: + n2 += z.execute("update y2027_tuimian_2027 set basis='roster_derived', " + "note=note||';页面名单分组行数(补记 basis)' " + "where trim(basis)='' and local_file=?", (f,)).rowcount +print('HTML 原件补 basis=%d 行' % n2) +z.execute('INSERT INTO changelog VALUES (?,?,?,?,?,?)', + (time.strftime('%Y-%m-%d %H:%M:%S'), 'basis_fix_20261101', + 'y2027_tuimian_2027', 'update', n + n2, + '0856定向批解析器只出 person_count 没带 basis,按原件类型补记推导口径')) +z.commit() +print('仍空=%d' % z.execute("select count(*) from y2027_tuimian_2027 where trim(basis)=''").fetchone()[0]) +print('basis 分布=%s' % (z.execute('select basis,count(*) from y2027_tuimian_2027 group by 1').fetchall(),)) +z.close() diff --git a/脚本/名单解析/fix_tm0856_quality.py b/脚本/名单解析/fix_tm0856_quality.py new file mode 100644 index 000000000..73a45eae9 --- /dev/null +++ b/脚本/名单解析/fix_tm0856_quality.py @@ -0,0 +1,53 @@ +# -*- coding: utf-8 -*- +"""撤掉 0856 批里质量不过关的 PDF 行,并修页面行的代码名称连写。 + +判据不是主观感觉:专业名等于专业代码、或名称是「页,共」「式」这类页眉碎片, +说明解析器把代码列/页眉当成了名称列。江西师范大学 58 行、浙江工业大学 3 行 +都属此类,占该批 PDF 的 40%,整批撤出不入库——错值比缺值更糟。 +页面 HTML 那 93 行可疑为 0,保留,但把「0806冶金工程」这种代码与名称连写的 +拆成名称,4 位学科码不塞进 6 位专业码列。 +""" +import re +import sqlite3 +import time + +ZX = r'D:/择校/数据库/zexiao_2027.db' +z = sqlite3.connect(ZX, timeout=400) +z.execute('PRAGMA busy_timeout=400000') +now = time.strftime('%Y-%m-%d %H:%M:%S') + +pdfs = [r[0] for r in z.execute( + "select distinct local_file from y2027_tuimian_2027 where local_file like '%.pdf' " + "and note like '%0856定向批%'")] +print('待撤 PDF 原件:', pdfs) +tot = 0 +for f in pdfs: + tot += z.execute("delete from y2027_tuimian_2027 where local_file=?", (f,)).rowcount +print('撤出=%d 行(质量不过关,未入交付)' % tot) + +# 页面行:名称里带前导 4 位码的拆开 +bad = z.execute("select rowid,major_name,major_code from y2027_tuimian_2027 " + "where local_file like '%.html' and major_name glob '[0-9][0-9][0-9][0-9]*'").fetchall() +fixed = 0 +for rid, nm, code in bad: + m = re.match(r'^(\d{4})(.+)$', nm or '') + if not m: + continue + z.execute("update y2027_tuimian_2027 set major_name=?, " + "note=note||';原表代码与名称连写,4位学科码不入专业码列', " + "basis='roster_derived' where rowid=?", + (m.group(2).strip(), rid)) + fixed += 1 +print('页面行拆连写=%d' % fixed) + +z.execute('INSERT INTO changelog VALUES (?,?,?,?,?,?)', + (now, 'tm0856_quality_fix_20261101', 'y2027_tuimian_2027', 'delete', tot, + '0856批PDF有40%%行专业名等于代码或取到页眉碎片,整批撤出;另修页面行连写%d处' % fixed)) +z.commit() +print('剩余=%d 行 / %d 校' % ( + z.execute('select count(*),count(distinct school) from y2027_tuimian_2027').fetchone())) +print('空basis=%d' % z.execute("select count(*) from y2027_tuimian_2027 where trim(basis)=''").fetchone()[0]) +print('名等于码=%d' % z.execute( + "select count(*) from y2027_tuimian_2027 where major_name=major_code or major_name glob '[0-9]*'").fetchone()[0]) +print('quick_check=', z.execute('PRAGMA quick_check').fetchone()[0]) +z.close() diff --git a/脚本/名单解析/postcheck_tm0856.py b/脚本/名单解析/postcheck_tm0856.py new file mode 100644 index 000000000..97c99ad54 --- /dev/null +++ b/脚本/名单解析/postcheck_tm0856.py @@ -0,0 +1,91 @@ +# -*- coding: utf-8 -*- +"""0856 定向批入库后核验 + 用新数据重算统考名额与推免占比。""" +import csv +import os +import re +import sqlite3 + +ZX = r'D:/择校/数据库/zexiao_2027.db' +OUT = r'D:/择校/审计报告/0856_统考名额与推免占比_20261101.csv' +LOG = r'D:/择校/运行暂存/agent_ingest_20261101/postcheck_tm0856.txt' + +z = sqlite3.connect(ZX, timeout=500) +z.execute('PRAGMA busy_timeout=500000') +L = [] +p = lambda s: L.append(str(s)) + +p('=== y2027_tuimian_2027 入库后核 ===') +p('键/校/人: %s' % (z.execute( + 'select count(*),count(distinct school),sum(cast(value as int)) from y2027_tuimian_2027').fetchone(),)) +p('值域: %s' % (z.execute( + 'select min(cast(value as int)),max(cast(value as int)) from y2027_tuimian_2027').fetchone(),)) +p('空专业键: %d' % z.execute( + "select count(*) from y2027_tuimian_2027 where trim(major_code)='' and trim(major_name)=''").fetchone()[0]) +p('缺来源: %d' % z.execute( + "select count(*) from y2027_tuimian_2027 where trim(source_url)=''").fetchone()[0]) +p('重复业务键组: %d' % z.execute( + 'select count(*) from (select 1 from y2027_tuimian_2027 group by school,year,college,' + 'college_code,major_code,major_name,kind,basis,plan_type having count(*)>1)').fetchone()[0]) +p('标 exceeds_quota: %d' % z.execute( + "select count(*) from y2027_tuimian_2027 where data_flag like '%exceeds_quota%'").fetchone()[0]) +p('basis 分布: %s' % (z.execute( + 'select basis,count(*) from y2027_tuimian_2027 group by 1').fetchall(),)) +p('校清单: %s' % ([r[0] for r in z.execute( + 'select distinct school from y2027_tuimian_2027 order by 1')],)) + +tm = {} +for s, c, v in z.execute("select school,major_code,value from y2027_tuimian_2027 where trim(major_code)<>''"): + if re.match(r'^\d+$', str(v)): + tm[(s, c)] = tm.get((s, c), 0) + int(v) +for s, c, v in z.execute("select school,major_code,received_recommend_count " + "from y2027_tuimian_major where trim(major_code)<>''"): + if re.match(r'^\d+$', str(v)): + tm[(s, c)] = tm.get((s, c), 0) + int(v) + +groups = {} +for dwmc, zydm, yx, s in z.execute( + "select dwmc,zydm,yxsmc,enroll_str from chsi_2027_dir " + "where zydm like '0856%' and trim(coalesce(enroll_str,''))<>''"): + k = (dwmc, zydm, (yx or '').strip()) + mp = re.search(r'专业:(\d+)', s) + mf = re.findall(r'研究方向:(\d+)', s) + if mp: + groups.setdefault(k, {'kind': '专业级', 'vals': []})['vals'].append(int(mp.group(1))) + elif mf: + groups.setdefault(k, {'kind': '方向级SUM', 'vals': []})['vals'] += [int(x) for x in mf] + +per = {} +for (dwmc, zydm, yx), g in groups.items(): + v = max(g['vals']) if g['kind'] == '专业级' else sum(g['vals']) + e = per.setdefault((dwmc, zydm), {'t': 0, 'k': set(), 'n': 0}) + e['t'] += v + e['k'].add(g['kind']) + e['n'] += 1 + +rows = [] +for (sch, code), e in sorted(per.items()): + t = tm.get((sch, code)) + if t is None: + continue + nm = z.execute("select zymc from chsi_2027_dir where dwmc=? and zydm=? limit 1", + (sch, code)).fetchone() + tot = e['t'] + t + rows.append(dict(院校=sch, 专业代码=code, 专业名称=nm[0] if nm else '', + 统考名额_不含推免=e['t'], 推免已接收=t, 总规模=tot, + 推免占比='%.0f%%' % (100.0 * t / tot if tot else 0), + 学院组数=e['n'], N口径='+'.join(sorted(e['k'])))) +with open(OUT, 'w', newline='', encoding='utf-8-sig') as f: + w = csv.DictWriter(f, fieldnames=list(rows[0].keys())) + w.writeheader() + w.writerows(rows) +p('') +p('=== 0856 占比表 ===') +p('键=%d 校=%d 写入 %s' % (len(rows), len({r['院校'] for r in rows}), os.path.basename(OUT))) +for r in sorted(rows, key=lambda x: -x['推免已接收'])[:12]: + p(' %-12s %s %-16s 统考=%-5d 推免=%-4d 占比=%-5s 组=%d' % ( + r['院校'], r['专业代码'], r['专业名称'][:16], r['统考名额_不含推免'], + r['推免已接收'], r['推免占比'], r['学院组数'])) +p('quick_check= %s' % z.execute('PRAGMA quick_check').fetchone()[0]) +z.close() +open(LOG, 'w', encoding='utf-8').write('\n'.join(L) + '\n') +print('\n'.join(L))