纠正0856定向批脏数据:撤152行并修连写
复核自己刚入库的行,发现专业名等于专业代码(江西师范58行、浙江工业3行, 占该批PDF 40%),还混进「页,共」「式」这类页眉碎片——解析器把代码列 和页眉当成了名称列。这批改判据的错不该由交付表承担,152行整批撤出, 撤前已备份并回读校验(152行,SHA e10bd009)。页面那93行可疑为0,保留, 另修8处「0806冶金工程」式连写,4位学科码不塞进6位专业码列。 撤后空basis=0、名等于码=0、缺来源=0,余1529行18校。 另纠正一条判断失误:两次 exit 124 是我把命令超时设得比实际执行短, 不是数据库写锁——活儿其实跑完了。
This commit is contained in:
1 parent
f7b95e5e7f
commit
bcd0f3c1c1
3 files changed
+187
No files matched your search
@@ -0,0 +1,43 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""补 basis 缺失并查清每条的出处,不是一刀切填 roster_derived。
|
||||
|
||||
y2027_tuimian_2027 靠 basis 区分「官方计数表」与「名单分组推导」,
|
||||
空着等于让下游无法判断这个数能不能直接当接收人数用。
|
||||
"""
|
||||
import sqlite3
|
||||
import time
|
||||
|
||||
ZX = r'D:/择校/数据库/zexiao_2027.db'
|
||||
z = sqlite3.connect(ZX, timeout=400)
|
||||
z.execute('PRAGMA busy_timeout=400000')
|
||||
|
||||
bad = z.execute("select local_file,source_url,count(*) from y2027_tuimian_2027 "
|
||||
"where trim(basis)='' group by 1,2 order by 3 desc").fetchall()
|
||||
print('basis 空涉及原件:')
|
||||
for r in bad:
|
||||
print(' %-34s %s 行=%d' % (str(r[0])[:34], str(r[1])[-28:], r[2]))
|
||||
|
||||
# 名单分组型:原件是逐人公示(PDF 名含 nlq/md 且无计数列);
|
||||
# 这里按解析器实际产出路径判:cand_batch(PDF 走 parse_official_roster_stat)只有名单分组一种产出
|
||||
pdfc = [r[0] for r in bad if (r[0] or '').endswith('.pdf')]
|
||||
htmlc = [r[0] for r in bad if (r[0] or '').endswith('.html')]
|
||||
n = 0
|
||||
for f in pdfc:
|
||||
n += z.execute("update y2027_tuimian_2027 set basis='roster_derived', "
|
||||
"note=note||';名单分组行数(补记 basis)' "
|
||||
"where trim(basis)='' and local_file=? and note like '%0856定向批%'", (f,)).rowcount
|
||||
print('PDF 原件补 basis=%d 行' % n)
|
||||
n2 = 0
|
||||
for f in htmlc:
|
||||
n2 += z.execute("update y2027_tuimian_2027 set basis='roster_derived', "
|
||||
"note=note||';页面名单分组行数(补记 basis)' "
|
||||
"where trim(basis)='' and local_file=?", (f,)).rowcount
|
||||
print('HTML 原件补 basis=%d 行' % n2)
|
||||
z.execute('INSERT INTO changelog VALUES (?,?,?,?,?,?)',
|
||||
(time.strftime('%Y-%m-%d %H:%M:%S'), 'basis_fix_20261101',
|
||||
'y2027_tuimian_2027', 'update', n + n2,
|
||||
'0856定向批解析器只出 person_count 没带 basis,按原件类型补记推导口径'))
|
||||
z.commit()
|
||||
print('仍空=%d' % z.execute("select count(*) from y2027_tuimian_2027 where trim(basis)=''").fetchone()[0])
|
||||
print('basis 分布=%s' % (z.execute('select basis,count(*) from y2027_tuimian_2027 group by 1').fetchall(),))
|
||||
z.close()
|
||||
@@ -0,0 +1,53 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""撤掉 0856 批里质量不过关的 PDF 行,并修页面行的代码名称连写。
|
||||
|
||||
判据不是主观感觉:专业名等于专业代码、或名称是「页,共」「式」这类页眉碎片,
|
||||
说明解析器把代码列/页眉当成了名称列。江西师范大学 58 行、浙江工业大学 3 行
|
||||
都属此类,占该批 PDF 的 40%,整批撤出不入库——错值比缺值更糟。
|
||||
页面 HTML 那 93 行可疑为 0,保留,但把「0806冶金工程」这种代码与名称连写的
|
||||
拆成名称,4 位学科码不塞进 6 位专业码列。
|
||||
"""
|
||||
import re
|
||||
import sqlite3
|
||||
import time
|
||||
|
||||
ZX = r'D:/择校/数据库/zexiao_2027.db'
|
||||
z = sqlite3.connect(ZX, timeout=400)
|
||||
z.execute('PRAGMA busy_timeout=400000')
|
||||
now = time.strftime('%Y-%m-%d %H:%M:%S')
|
||||
|
||||
pdfs = [r[0] for r in z.execute(
|
||||
"select distinct local_file from y2027_tuimian_2027 where local_file like '%.pdf' "
|
||||
"and note like '%0856定向批%'")]
|
||||
print('待撤 PDF 原件:', pdfs)
|
||||
tot = 0
|
||||
for f in pdfs:
|
||||
tot += z.execute("delete from y2027_tuimian_2027 where local_file=?", (f,)).rowcount
|
||||
print('撤出=%d 行(质量不过关,未入交付)' % tot)
|
||||
|
||||
# 页面行:名称里带前导 4 位码的拆开
|
||||
bad = z.execute("select rowid,major_name,major_code from y2027_tuimian_2027 "
|
||||
"where local_file like '%.html' and major_name glob '[0-9][0-9][0-9][0-9]*'").fetchall()
|
||||
fixed = 0
|
||||
for rid, nm, code in bad:
|
||||
m = re.match(r'^(\d{4})(.+)$', nm or '')
|
||||
if not m:
|
||||
continue
|
||||
z.execute("update y2027_tuimian_2027 set major_name=?, "
|
||||
"note=note||';原表代码与名称连写,4位学科码不入专业码列', "
|
||||
"basis='roster_derived' where rowid=?",
|
||||
(m.group(2).strip(), rid))
|
||||
fixed += 1
|
||||
print('页面行拆连写=%d' % fixed)
|
||||
|
||||
z.execute('INSERT INTO changelog VALUES (?,?,?,?,?,?)',
|
||||
(now, 'tm0856_quality_fix_20261101', 'y2027_tuimian_2027', 'delete', tot,
|
||||
'0856批PDF有40%%行专业名等于代码或取到页眉碎片,整批撤出;另修页面行连写%d处' % fixed))
|
||||
z.commit()
|
||||
print('剩余=%d 行 / %d 校' % (
|
||||
z.execute('select count(*),count(distinct school) from y2027_tuimian_2027').fetchone()))
|
||||
print('空basis=%d' % z.execute("select count(*) from y2027_tuimian_2027 where trim(basis)=''").fetchone()[0])
|
||||
print('名等于码=%d' % z.execute(
|
||||
"select count(*) from y2027_tuimian_2027 where major_name=major_code or major_name glob '[0-9]*'").fetchone()[0])
|
||||
print('quick_check=', z.execute('PRAGMA quick_check').fetchone()[0])
|
||||
z.close()
|
||||
@@ -0,0 +1,91 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""0856 定向批入库后核验 + 用新数据重算统考名额与推免占比。"""
|
||||
import csv
|
||||
import os
|
||||
import re
|
||||
import sqlite3
|
||||
|
||||
ZX = r'D:/择校/数据库/zexiao_2027.db'
|
||||
OUT = r'D:/择校/审计报告/0856_统考名额与推免占比_20261101.csv'
|
||||
LOG = r'D:/择校/运行暂存/agent_ingest_20261101/postcheck_tm0856.txt'
|
||||
|
||||
z = sqlite3.connect(ZX, timeout=500)
|
||||
z.execute('PRAGMA busy_timeout=500000')
|
||||
L = []
|
||||
p = lambda s: L.append(str(s))
|
||||
|
||||
p('=== y2027_tuimian_2027 入库后核 ===')
|
||||
p('键/校/人: %s' % (z.execute(
|
||||
'select count(*),count(distinct school),sum(cast(value as int)) from y2027_tuimian_2027').fetchone(),))
|
||||
p('值域: %s' % (z.execute(
|
||||
'select min(cast(value as int)),max(cast(value as int)) from y2027_tuimian_2027').fetchone(),))
|
||||
p('空专业键: %d' % z.execute(
|
||||
"select count(*) from y2027_tuimian_2027 where trim(major_code)='' and trim(major_name)=''").fetchone()[0])
|
||||
p('缺来源: %d' % z.execute(
|
||||
"select count(*) from y2027_tuimian_2027 where trim(source_url)=''").fetchone()[0])
|
||||
p('重复业务键组: %d' % z.execute(
|
||||
'select count(*) from (select 1 from y2027_tuimian_2027 group by school,year,college,'
|
||||
'college_code,major_code,major_name,kind,basis,plan_type having count(*)>1)').fetchone()[0])
|
||||
p('标 exceeds_quota: %d' % z.execute(
|
||||
"select count(*) from y2027_tuimian_2027 where data_flag like '%exceeds_quota%'").fetchone()[0])
|
||||
p('basis 分布: %s' % (z.execute(
|
||||
'select basis,count(*) from y2027_tuimian_2027 group by 1').fetchall(),))
|
||||
p('校清单: %s' % ([r[0] for r in z.execute(
|
||||
'select distinct school from y2027_tuimian_2027 order by 1')],))
|
||||
|
||||
tm = {}
|
||||
for s, c, v in z.execute("select school,major_code,value from y2027_tuimian_2027 where trim(major_code)<>''"):
|
||||
if re.match(r'^\d+$', str(v)):
|
||||
tm[(s, c)] = tm.get((s, c), 0) + int(v)
|
||||
for s, c, v in z.execute("select school,major_code,received_recommend_count "
|
||||
"from y2027_tuimian_major where trim(major_code)<>''"):
|
||||
if re.match(r'^\d+$', str(v)):
|
||||
tm[(s, c)] = tm.get((s, c), 0) + int(v)
|
||||
|
||||
groups = {}
|
||||
for dwmc, zydm, yx, s in z.execute(
|
||||
"select dwmc,zydm,yxsmc,enroll_str from chsi_2027_dir "
|
||||
"where zydm like '0856%' and trim(coalesce(enroll_str,''))<>''"):
|
||||
k = (dwmc, zydm, (yx or '').strip())
|
||||
mp = re.search(r'专业:(\d+)', s)
|
||||
mf = re.findall(r'研究方向:(\d+)', s)
|
||||
if mp:
|
||||
groups.setdefault(k, {'kind': '专业级', 'vals': []})['vals'].append(int(mp.group(1)))
|
||||
elif mf:
|
||||
groups.setdefault(k, {'kind': '方向级SUM', 'vals': []})['vals'] += [int(x) for x in mf]
|
||||
|
||||
per = {}
|
||||
for (dwmc, zydm, yx), g in groups.items():
|
||||
v = max(g['vals']) if g['kind'] == '专业级' else sum(g['vals'])
|
||||
e = per.setdefault((dwmc, zydm), {'t': 0, 'k': set(), 'n': 0})
|
||||
e['t'] += v
|
||||
e['k'].add(g['kind'])
|
||||
e['n'] += 1
|
||||
|
||||
rows = []
|
||||
for (sch, code), e in sorted(per.items()):
|
||||
t = tm.get((sch, code))
|
||||
if t is None:
|
||||
continue
|
||||
nm = z.execute("select zymc from chsi_2027_dir where dwmc=? and zydm=? limit 1",
|
||||
(sch, code)).fetchone()
|
||||
tot = e['t'] + t
|
||||
rows.append(dict(院校=sch, 专业代码=code, 专业名称=nm[0] if nm else '',
|
||||
统考名额_不含推免=e['t'], 推免已接收=t, 总规模=tot,
|
||||
推免占比='%.0f%%' % (100.0 * t / tot if tot else 0),
|
||||
学院组数=e['n'], N口径='+'.join(sorted(e['k']))))
|
||||
with open(OUT, 'w', newline='', encoding='utf-8-sig') as f:
|
||||
w = csv.DictWriter(f, fieldnames=list(rows[0].keys()))
|
||||
w.writeheader()
|
||||
w.writerows(rows)
|
||||
p('')
|
||||
p('=== 0856 占比表 ===')
|
||||
p('键=%d 校=%d 写入 %s' % (len(rows), len({r['院校'] for r in rows}), os.path.basename(OUT)))
|
||||
for r in sorted(rows, key=lambda x: -x['推免已接收'])[:12]:
|
||||
p(' %-12s %s %-16s 统考=%-5d 推免=%-4d 占比=%-5s 组=%d' % (
|
||||
r['院校'], r['专业代码'], r['专业名称'][:16], r['统考名额_不含推免'],
|
||||
r['推免已接收'], r['推免占比'], r['学院组数']))
|
||||
p('quick_check= %s' % z.execute('PRAGMA quick_check').fetchone()[0])
|
||||
z.close()
|
||||
open(LOG, 'w', encoding='utf-8').write('\n'.join(L) + '\n')
|
||||
print('\n'.join(L))
|
||||
Reference in new issue
Block a user