纠正0856定向批脏数据:撤152行并修连写

复核自己刚入库的行,发现专业名等于专业代码(江西师范58行、浙江工业3行,
占该批PDF 40%),还混进「页,共」「式」这类页眉碎片——解析器把代码列
和页眉当成了名称列。这批改判据的错不该由交付表承担,152行整批撤出,
撤前已备份并回读校验(152行,SHA e10bd009)。页面那93行可疑为0,保留,
另修8处「0806冶金工程」式连写,4位学科码不塞进6位专业码列。

撤后空basis=0、名等于码=0、缺来源=0,余1529行18校。
另纠正一条判断失误:两次 exit 124 是我把命令超时设得比实际执行短,
不是数据库写锁——活儿其实跑完了。
This commit is contained in:
Your Name committed 2026-11-02 13:52:18 +08:00
1 parent f7b95e5e7f
commit bcd0f3c1c1
3 files changed
+187

No files matched your search

+43
View File
@@ -0,0 +1,43 @@
# -*- coding: utf-8 -*-
"""补 basis 缺失并查清每条的出处,不是一刀切填 roster_derived。
y2027_tuimian_2027 靠 basis 区分「官方计数表」与「名单分组推导」,
空着等于让下游无法判断这个数能不能直接当接收人数用。
"""
import sqlite3
import time
ZX = r'D:/择校/数据库/zexiao_2027.db'
z = sqlite3.connect(ZX, timeout=400)
z.execute('PRAGMA busy_timeout=400000')
bad = z.execute("select local_file,source_url,count(*) from y2027_tuimian_2027 "
"where trim(basis)='' group by 1,2 order by 3 desc").fetchall()
print('basis 空涉及原件:')
for r in bad:
print(' %-34s %s 行=%d' % (str(r[0])[:34], str(r[1])[-28:], r[2]))
# 名单分组型:原件是逐人公示(PDF 名含 nlq/md 且无计数列);
# 这里按解析器实际产出路径判:cand_batch(PDF 走 parse_official_roster_stat)只有名单分组一种产出
pdfc = [r[0] for r in bad if (r[0] or '').endswith('.pdf')]
htmlc = [r[0] for r in bad if (r[0] or '').endswith('.html')]
n = 0
for f in pdfc:
n += z.execute("update y2027_tuimian_2027 set basis='roster_derived', "
"note=note||';名单分组行数(补记 basis)' "
"where trim(basis)='' and local_file=? and note like '%0856定向批%'", (f,)).rowcount
print('PDF 原件补 basis=%d 行' % n)
n2 = 0
for f in htmlc:
n2 += z.execute("update y2027_tuimian_2027 set basis='roster_derived', "
"note=note||';页面名单分组行数(补记 basis)' "
"where trim(basis)='' and local_file=?", (f,)).rowcount
print('HTML 原件补 basis=%d 行' % n2)
z.execute('INSERT INTO changelog VALUES (?,?,?,?,?,?)',
(time.strftime('%Y-%m-%d %H:%M:%S'), 'basis_fix_20261101',
'y2027_tuimian_2027', 'update', n + n2,
'0856定向批解析器只出 person_count 没带 basis,按原件类型补记推导口径'))
z.commit()
print('仍空=%d' % z.execute("select count(*) from y2027_tuimian_2027 where trim(basis)=''").fetchone()[0])
print('basis 分布=%s' % (z.execute('select basis,count(*) from y2027_tuimian_2027 group by 1').fetchall(),))
z.close()
+53
View File
@@ -0,0 +1,53 @@
# -*- coding: utf-8 -*-
"""撤掉 0856 批里质量不过关的 PDF 行,并修页面行的代码名称连写。
判据不是主观感觉:专业名等于专业代码、或名称是「页,共」「式」这类页眉碎片,
说明解析器把代码列/页眉当成了名称列。江西师范大学 58 行、浙江工业大学 3 行
都属此类,占该批 PDF 的 40%,整批撤出不入库——错值比缺值更糟。
页面 HTML 那 93 行可疑为 0,保留,但把「0806冶金工程」这种代码与名称连写的
拆成名称,4 位学科码不塞进 6 位专业码列。
"""
import re
import sqlite3
import time
ZX = r'D:/择校/数据库/zexiao_2027.db'
z = sqlite3.connect(ZX, timeout=400)
z.execute('PRAGMA busy_timeout=400000')
now = time.strftime('%Y-%m-%d %H:%M:%S')
pdfs = [r[0] for r in z.execute(
"select distinct local_file from y2027_tuimian_2027 where local_file like '%.pdf' "
"and note like '%0856定向批%'")]
print('待撤 PDF 原件:', pdfs)
tot = 0
for f in pdfs:
tot += z.execute("delete from y2027_tuimian_2027 where local_file=?", (f,)).rowcount
print('撤出=%d 行(质量不过关,未入交付)' % tot)
# 页面行:名称里带前导 4 位码的拆开
bad = z.execute("select rowid,major_name,major_code from y2027_tuimian_2027 "
"where local_file like '%.html' and major_name glob '[0-9][0-9][0-9][0-9]*'").fetchall()
fixed = 0
for rid, nm, code in bad:
m = re.match(r'^(\d{4})(.+)$', nm or '')
if not m:
continue
z.execute("update y2027_tuimian_2027 set major_name=?, "
"note=note||';原表代码与名称连写,4位学科码不入专业码列', "
"basis='roster_derived' where rowid=?",
(m.group(2).strip(), rid))
fixed += 1
print('页面行拆连写=%d' % fixed)
z.execute('INSERT INTO changelog VALUES (?,?,?,?,?,?)',
(now, 'tm0856_quality_fix_20261101', 'y2027_tuimian_2027', 'delete', tot,
'0856批PDF有40%%行专业名等于代码或取到页眉碎片,整批撤出;另修页面行连写%d处' % fixed))
z.commit()
print('剩余=%d 行 / %d 校' % (
z.execute('select count(*),count(distinct school) from y2027_tuimian_2027').fetchone()))
print('空basis=%d' % z.execute("select count(*) from y2027_tuimian_2027 where trim(basis)=''").fetchone()[0])
print('名等于码=%d' % z.execute(
"select count(*) from y2027_tuimian_2027 where major_name=major_code or major_name glob '[0-9]*'").fetchone()[0])
print('quick_check=', z.execute('PRAGMA quick_check').fetchone()[0])
z.close()
+91
View File
@@ -0,0 +1,91 @@
# -*- coding: utf-8 -*-
"""0856 定向批入库后核验 + 用新数据重算统考名额与推免占比。"""
import csv
import os
import re
import sqlite3
ZX = r'D:/择校/数据库/zexiao_2027.db'
OUT = r'D:/择校/审计报告/0856_统考名额与推免占比_20261101.csv'
LOG = r'D:/择校/运行暂存/agent_ingest_20261101/postcheck_tm0856.txt'
z = sqlite3.connect(ZX, timeout=500)
z.execute('PRAGMA busy_timeout=500000')
L = []
p = lambda s: L.append(str(s))
p('=== y2027_tuimian_2027 入库后核 ===')
p('键/校/人: %s' % (z.execute(
'select count(*),count(distinct school),sum(cast(value as int)) from y2027_tuimian_2027').fetchone(),))
p('值域: %s' % (z.execute(
'select min(cast(value as int)),max(cast(value as int)) from y2027_tuimian_2027').fetchone(),))
p('空专业键: %d' % z.execute(
"select count(*) from y2027_tuimian_2027 where trim(major_code)='' and trim(major_name)=''").fetchone()[0])
p('缺来源: %d' % z.execute(
"select count(*) from y2027_tuimian_2027 where trim(source_url)=''").fetchone()[0])
p('重复业务键组: %d' % z.execute(
'select count(*) from (select 1 from y2027_tuimian_2027 group by school,year,college,'
'college_code,major_code,major_name,kind,basis,plan_type having count(*)>1)').fetchone()[0])
p('标 exceeds_quota: %d' % z.execute(
"select count(*) from y2027_tuimian_2027 where data_flag like '%exceeds_quota%'").fetchone()[0])
p('basis 分布: %s' % (z.execute(
'select basis,count(*) from y2027_tuimian_2027 group by 1').fetchall(),))
p('校清单: %s' % ([r[0] for r in z.execute(
'select distinct school from y2027_tuimian_2027 order by 1')],))
tm = {}
for s, c, v in z.execute("select school,major_code,value from y2027_tuimian_2027 where trim(major_code)<>''"):
if re.match(r'^\d+$', str(v)):
tm[(s, c)] = tm.get((s, c), 0) + int(v)
for s, c, v in z.execute("select school,major_code,received_recommend_count "
"from y2027_tuimian_major where trim(major_code)<>''"):
if re.match(r'^\d+$', str(v)):
tm[(s, c)] = tm.get((s, c), 0) + int(v)
groups = {}
for dwmc, zydm, yx, s in z.execute(
"select dwmc,zydm,yxsmc,enroll_str from chsi_2027_dir "
"where zydm like '0856%' and trim(coalesce(enroll_str,''))<>''"):
k = (dwmc, zydm, (yx or '').strip())
mp = re.search(r'专业:(\d+)', s)
mf = re.findall(r'研究方向:(\d+)', s)
if mp:
groups.setdefault(k, {'kind': '专业级', 'vals': []})['vals'].append(int(mp.group(1)))
elif mf:
groups.setdefault(k, {'kind': '方向级SUM', 'vals': []})['vals'] += [int(x) for x in mf]
per = {}
for (dwmc, zydm, yx), g in groups.items():
v = max(g['vals']) if g['kind'] == '专业级' else sum(g['vals'])
e = per.setdefault((dwmc, zydm), {'t': 0, 'k': set(), 'n': 0})
e['t'] += v
e['k'].add(g['kind'])
e['n'] += 1
rows = []
for (sch, code), e in sorted(per.items()):
t = tm.get((sch, code))
if t is None:
continue
nm = z.execute("select zymc from chsi_2027_dir where dwmc=? and zydm=? limit 1",
(sch, code)).fetchone()
tot = e['t'] + t
rows.append(dict(院校=sch, 专业代码=code, 专业名称=nm[0] if nm else '',
统考名额_不含推免=e['t'], 推免已接收=t, 总规模=tot,
推免占比='%.0f%%' % (100.0 * t / tot if tot else 0),
学院组数=e['n'], N口径='+'.join(sorted(e['k']))))
with open(OUT, 'w', newline='', encoding='utf-8-sig') as f:
w = csv.DictWriter(f, fieldnames=list(rows[0].keys()))
w.writeheader()
w.writerows(rows)
p('')
p('=== 0856 占比表 ===')
p('键=%d 校=%d 写入 %s' % (len(rows), len({r['院校'] for r in rows}), os.path.basename(OUT)))
for r in sorted(rows, key=lambda x: -x['推免已接收'])[:12]:
p(' %-12s %s %-16s 统考=%-5d 推免=%-4d 占比=%-5s 组=%d' % (
r['院校'], r['专业代码'], r['专业名称'][:16], r['统考名额_不含推免'],
r['推免已接收'], r['推免占比'], r['学院组数']))
p('quick_check= %s' % z.execute('PRAGMA quick_check').fetchone()[0])
z.close()
open(LOG, 'w', encoding='utf-8').write('\n'.join(L) + '\n')
print('\n'.join(L))