归档633校简章学费学制产出并定下使用限制

agent交的这份3568行/633校,全部带 notice_key 证据、99.4%带URL,
与官方CHSI学费可比379例中317相等(83.6%),一致度高于启航(76.2%)。

但三条限制必须写死:
粒度上只有614行定位到专业码,1696行是校级标准、1258行是院系/学位级,
校级值当专业值用会误导;同校同年给出多个学费值的有393组;
住宿只有「提供/不安排」两值且没按学习方式拆,
比我自抽那135行粒度粗——正是我第一轮踩过的坑,合并前要回正文重判归属。
This commit is contained in:
Your Name committed 2026-11-01 02:25:52 +08:00
1 parent a319e90565
commit 4c96bc27f9
3 files changed
+3708

No files matched your search

@@ -0,0 +1,34 @@
{
"generated_at": "2026-11-01",
"rows": 3568,
"schools": 633,
"years": {
"2026": 2953,
"2027": 338,
"2025": 277
},
"grain": {
"校级": 1696,
"院系/学位级": 1258,
"专业级": 614
},
"有值字段": {
"tuition_year": 2256,
"tuition_total": 254,
"duration_year": 863,
"accommodation": 507
},
"证据完整": {
"带source_url": 3545,
"带notice_key": 3568
},
"官方交叉验证": "与 chsi_2027_dir 学费可比 379 例中 317 相等(83.6%),高于启航的 76.2%",
"使用限制": [
"校级/院系级值不能当专业值交付,入库表已按 grain 分列并标 not_major_level",
"同校同年多值 393 组标 school_multi_tuition,note 带 multi_ 的标 ambiguous_value,不参与交付排序",
"住宿只有「提供/不安排」两值且未按学习方式拆分,比我自抽的 135 行粒度粗,合并前需回到公告正文重新判归属"
],
"files": {
"stage_a_rows.csv": "f3a797c5f4e71246"
}
}
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,105 @@
# -*- coding: utf-8 -*-
"""简章抽出的学费/学制入库 web_tuition_rules,并与官方 CHSI 值交叉标注。
三件事必须分开处理,否则会把交付表带偏:
1. 粒度分列:1,696 行是校级标准、1,258 行是院系/学位类型级、只有 614 行定位到专业码,
校级值不能当专业值用,所以 grain 入列,交付层按粒度取。
2. 多值未消歧:同校同年给出多个学费值的有 393 组,164 行 note 里带 multi_ 标记,
这类标 ambiguous,不参与交付排序。
3. 与官方对账:官方 CHSI 有 2027 学费,可比 379 例中 317 相等(83.6%),
相等的标 cross_ok,差异的保留双方原值不覆盖。
"""
import csv
import json
import re
import sqlite3
import time
ZX = r'D:/择校/数据库/zexiao_2027.db'
SRC = r'D:/择校/运行暂存/agent_ingest_20261101/tuition/stage_a_rows.csv'
TABLE = 'web_tuition_rules'
DDL = '''CREATE TABLE IF NOT EXISTS %s (
"school" TEXT, "school_code" TEXT, "year" TEXT, "grain" TEXT,
"college" TEXT, "degree_type" TEXT, "study_mode" TEXT,
"major_code" TEXT, "major_name" TEXT,
"tuition_year" TEXT, "tuition_total" TEXT, "tuition_kind" TEXT,
"duration_year" TEXT, "accommodation" TEXT,
"source_url" TEXT, "notice_key" TEXT, "evidence" TEXT,
"data_flag" TEXT, "cross_check" TEXT, "note" TEXT, "_ingested_at" TEXT,
PRIMARY KEY ("school","year","grain","college","degree_type","study_mode",
"major_code","major_name")
)''' % TABLE
def main():
rows = list(csv.DictReader(open(SRC, encoding='utf-8-sig')))
z = sqlite3.connect(ZX, timeout=600)
z.execute('PRAGMA busy_timeout=600000')
z.execute(DDL)
chsi = {}
for s, c, v in z.execute("select dwmc,zydm,tuition_fee from chsi_2027_dir "
"where tuition_fee glob '[0-9]*'"):
chsi.setdefault((s, c), str(re.search(r'\d+', str(v)).group()))
per_school_year = {}
for r in rows:
if (r.get('tuition_year') or '').strip():
per_school_year.setdefault((r['school'], r['year']), set()).add(r['tuition_year'])
bad, ins = {}, 0
now = time.strftime('%Y-%m-%dT%H:%M:%S')
for r in rows:
if not (r.get('source_url') or '').startswith('http'):
bad['no_url'] = bad.get('no_url', 0) + 1
continue
ty, tt = (r.get('tuition_year') or '').strip(), (r.get('tuition_total') or '').strip()
for v in (ty, tt):
if v and not re.match(r'^\d{3,7}$', v):
bad['value_format'] = bad.get('value_format', 0) + 1
if not (ty or tt or (r.get('duration_year') or '').strip() or (r.get('accommodation') or '').strip()):
bad['empty_row'] = bad.get('empty_row', 0) + 1
continue
grain = ('专业级' if ty or tt or (r.get('major_code') or '').strip() else '')
if not grain:
grain = '校级' if not (r.get('college') or r.get('degree_type') or '').strip() else '院系/学位级'
if (r.get('major_code') or '').strip():
grain = '专业级'
flags = []
if 'multi_' in (r.get('note') or ''):
flags.append('ambiguous_value')
if len(per_school_year.get((r['school'], r['year']), ())) > 1 and grain != '专业级':
flags.append('school_multi_tuition')
if grain != '专业级':
flags.append('not_major_level')
cross = ''
if ty and (r.get('major_code') or '').strip():
v = chsi.get((r['school'], r['major_code']))
if v:
cross = 'official_same' if v == ty else 'official_diff:%s' % v
m = re.search(r'notice_key=([^ |]+)', r.get('local_evidence', ''))
cols = ['school', 'school_code', 'year', 'grain', 'college', 'degree_type',
'study_mode', 'major_code', 'major_name', 'tuition_year', 'tuition_total',
'tuition_kind', 'duration_year', 'accommodation', 'source_url',
'notice_key', 'evidence', 'data_flag', 'cross_check', 'note']
vals = [r.get('school', ''), r.get('school_code', ''), r.get('year', ''), grain,
r.get('college', ''), r.get('degree_type', ''), r.get('study_mode', ''),
r.get('major_code', ''), r.get('major_name', ''), ty, tt,
r.get('tuition_kind', ''), r.get('duration_year', ''),
r.get('accommodation', ''), r.get('source_url', ''),
m.group(1) if m else '', r.get('local_evidence', ''),
','.join(flags), cross, r.get('note', '')]
ins += z.execute('INSERT OR REPLACE INTO %s ("%s","_ingested_at") VALUES (%s,?)' % (
TABLE, '","'.join(cols), ','.join(['?'] * len(cols))), vals + [now]).rowcount
z.execute('INSERT INTO changelog VALUES (?,?,?,?,?,?)',
(time.strftime('%Y-%m-%d %H:%M:%S'), 'web_tuition_rules_20261101', TABLE,
'create+insert', ins, '简章学费学制入库;粒度分列,官方CHSI交叉标注;拒绝=%s' % bad))
z.commit()
print('写入=%d 拒绝=%s 表内=%d 校=%d' % (ins, bad,
z.execute('select count(*) from %s' % TABLE).fetchone()[0],
z.execute('select count(distinct school) from %s' % TABLE).fetchone()[0]))
print('粒度:', z.execute('select grain,count(*) from %s group by 1' % TABLE).fetchall())
print('交叉:', z.execute("select cross_check,count(*) from %s group by 1 order by 2 desc limit 4" % TABLE).fetchall())
print('quick_check=', z.execute('PRAGMA quick_check').fetchone()[0])
z.close()
if __name__ == '__main__':
main()