归档633校简章学费学制产出并定下使用限制
agent交的这份3568行/633校,全部带 notice_key 证据、99.4%带URL, 与官方CHSI学费可比379例中317相等(83.6%),一致度高于启航(76.2%)。 但三条限制必须写死: 粒度上只有614行定位到专业码,1696行是校级标准、1258行是院系/学位级, 校级值当专业值用会误导;同校同年给出多个学费值的有393组; 住宿只有「提供/不安排」两值且没按学习方式拆, 比我自抽那135行粒度粗——正是我第一轮踩过的坑,合并前要回正文重判归属。
This commit is contained in:
1 parent
a319e90565
commit
4c96bc27f9
3 files changed
+3708
No files matched your search
@@ -0,0 +1,34 @@
|
||||
{
|
||||
"generated_at": "2026-11-01",
|
||||
"rows": 3568,
|
||||
"schools": 633,
|
||||
"years": {
|
||||
"2026": 2953,
|
||||
"2027": 338,
|
||||
"2025": 277
|
||||
},
|
||||
"grain": {
|
||||
"校级": 1696,
|
||||
"院系/学位级": 1258,
|
||||
"专业级": 614
|
||||
},
|
||||
"有值字段": {
|
||||
"tuition_year": 2256,
|
||||
"tuition_total": 254,
|
||||
"duration_year": 863,
|
||||
"accommodation": 507
|
||||
},
|
||||
"证据完整": {
|
||||
"带source_url": 3545,
|
||||
"带notice_key": 3568
|
||||
},
|
||||
"官方交叉验证": "与 chsi_2027_dir 学费可比 379 例中 317 相等(83.6%),高于启航的 76.2%",
|
||||
"使用限制": [
|
||||
"校级/院系级值不能当专业值交付,入库表已按 grain 分列并标 not_major_level",
|
||||
"同校同年多值 393 组标 school_multi_tuition,note 带 multi_ 的标 ambiguous_value,不参与交付排序",
|
||||
"住宿只有「提供/不安排」两值且未按学习方式拆分,比我自抽的 135 行粒度粗,合并前需回到公告正文重新判归属"
|
||||
],
|
||||
"files": {
|
||||
"stage_a_rows.csv": "f3a797c5f4e71246"
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,105 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""简章抽出的学费/学制入库 web_tuition_rules,并与官方 CHSI 值交叉标注。
|
||||
|
||||
三件事必须分开处理,否则会把交付表带偏:
|
||||
1. 粒度分列:1,696 行是校级标准、1,258 行是院系/学位类型级、只有 614 行定位到专业码,
|
||||
校级值不能当专业值用,所以 grain 入列,交付层按粒度取。
|
||||
2. 多值未消歧:同校同年给出多个学费值的有 393 组,164 行 note 里带 multi_ 标记,
|
||||
这类标 ambiguous,不参与交付排序。
|
||||
3. 与官方对账:官方 CHSI 有 2027 学费,可比 379 例中 317 相等(83.6%),
|
||||
相等的标 cross_ok,差异的保留双方原值不覆盖。
|
||||
"""
|
||||
import csv
|
||||
import json
|
||||
import re
|
||||
import sqlite3
|
||||
import time
|
||||
|
||||
ZX = r'D:/择校/数据库/zexiao_2027.db'
|
||||
SRC = r'D:/择校/运行暂存/agent_ingest_20261101/tuition/stage_a_rows.csv'
|
||||
TABLE = 'web_tuition_rules'
|
||||
DDL = '''CREATE TABLE IF NOT EXISTS %s (
|
||||
"school" TEXT, "school_code" TEXT, "year" TEXT, "grain" TEXT,
|
||||
"college" TEXT, "degree_type" TEXT, "study_mode" TEXT,
|
||||
"major_code" TEXT, "major_name" TEXT,
|
||||
"tuition_year" TEXT, "tuition_total" TEXT, "tuition_kind" TEXT,
|
||||
"duration_year" TEXT, "accommodation" TEXT,
|
||||
"source_url" TEXT, "notice_key" TEXT, "evidence" TEXT,
|
||||
"data_flag" TEXT, "cross_check" TEXT, "note" TEXT, "_ingested_at" TEXT,
|
||||
PRIMARY KEY ("school","year","grain","college","degree_type","study_mode",
|
||||
"major_code","major_name")
|
||||
)''' % TABLE
|
||||
|
||||
|
||||
def main():
|
||||
rows = list(csv.DictReader(open(SRC, encoding='utf-8-sig')))
|
||||
z = sqlite3.connect(ZX, timeout=600)
|
||||
z.execute('PRAGMA busy_timeout=600000')
|
||||
z.execute(DDL)
|
||||
chsi = {}
|
||||
for s, c, v in z.execute("select dwmc,zydm,tuition_fee from chsi_2027_dir "
|
||||
"where tuition_fee glob '[0-9]*'"):
|
||||
chsi.setdefault((s, c), str(re.search(r'\d+', str(v)).group()))
|
||||
per_school_year = {}
|
||||
for r in rows:
|
||||
if (r.get('tuition_year') or '').strip():
|
||||
per_school_year.setdefault((r['school'], r['year']), set()).add(r['tuition_year'])
|
||||
bad, ins = {}, 0
|
||||
now = time.strftime('%Y-%m-%dT%H:%M:%S')
|
||||
for r in rows:
|
||||
if not (r.get('source_url') or '').startswith('http'):
|
||||
bad['no_url'] = bad.get('no_url', 0) + 1
|
||||
continue
|
||||
ty, tt = (r.get('tuition_year') or '').strip(), (r.get('tuition_total') or '').strip()
|
||||
for v in (ty, tt):
|
||||
if v and not re.match(r'^\d{3,7}$', v):
|
||||
bad['value_format'] = bad.get('value_format', 0) + 1
|
||||
if not (ty or tt or (r.get('duration_year') or '').strip() or (r.get('accommodation') or '').strip()):
|
||||
bad['empty_row'] = bad.get('empty_row', 0) + 1
|
||||
continue
|
||||
grain = ('专业级' if ty or tt or (r.get('major_code') or '').strip() else '')
|
||||
if not grain:
|
||||
grain = '校级' if not (r.get('college') or r.get('degree_type') or '').strip() else '院系/学位级'
|
||||
if (r.get('major_code') or '').strip():
|
||||
grain = '专业级'
|
||||
flags = []
|
||||
if 'multi_' in (r.get('note') or ''):
|
||||
flags.append('ambiguous_value')
|
||||
if len(per_school_year.get((r['school'], r['year']), ())) > 1 and grain != '专业级':
|
||||
flags.append('school_multi_tuition')
|
||||
if grain != '专业级':
|
||||
flags.append('not_major_level')
|
||||
cross = ''
|
||||
if ty and (r.get('major_code') or '').strip():
|
||||
v = chsi.get((r['school'], r['major_code']))
|
||||
if v:
|
||||
cross = 'official_same' if v == ty else 'official_diff:%s' % v
|
||||
m = re.search(r'notice_key=([^ |]+)', r.get('local_evidence', ''))
|
||||
cols = ['school', 'school_code', 'year', 'grain', 'college', 'degree_type',
|
||||
'study_mode', 'major_code', 'major_name', 'tuition_year', 'tuition_total',
|
||||
'tuition_kind', 'duration_year', 'accommodation', 'source_url',
|
||||
'notice_key', 'evidence', 'data_flag', 'cross_check', 'note']
|
||||
vals = [r.get('school', ''), r.get('school_code', ''), r.get('year', ''), grain,
|
||||
r.get('college', ''), r.get('degree_type', ''), r.get('study_mode', ''),
|
||||
r.get('major_code', ''), r.get('major_name', ''), ty, tt,
|
||||
r.get('tuition_kind', ''), r.get('duration_year', ''),
|
||||
r.get('accommodation', ''), r.get('source_url', ''),
|
||||
m.group(1) if m else '', r.get('local_evidence', ''),
|
||||
','.join(flags), cross, r.get('note', '')]
|
||||
ins += z.execute('INSERT OR REPLACE INTO %s ("%s","_ingested_at") VALUES (%s,?)' % (
|
||||
TABLE, '","'.join(cols), ','.join(['?'] * len(cols))), vals + [now]).rowcount
|
||||
z.execute('INSERT INTO changelog VALUES (?,?,?,?,?,?)',
|
||||
(time.strftime('%Y-%m-%d %H:%M:%S'), 'web_tuition_rules_20261101', TABLE,
|
||||
'create+insert', ins, '简章学费学制入库;粒度分列,官方CHSI交叉标注;拒绝=%s' % bad))
|
||||
z.commit()
|
||||
print('写入=%d 拒绝=%s 表内=%d 校=%d' % (ins, bad,
|
||||
z.execute('select count(*) from %s' % TABLE).fetchone()[0],
|
||||
z.execute('select count(distinct school) from %s' % TABLE).fetchone()[0]))
|
||||
print('粒度:', z.execute('select grain,count(*) from %s group by 1' % TABLE).fetchall())
|
||||
print('交叉:', z.execute("select cross_check,count(*) from %s group by 1 order by 2 desc limit 4" % TABLE).fetchall())
|
||||
print('quick_check=', z.execute('PRAGMA quick_check').fetchone()[0])
|
||||
z.close()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in new issue
Block a user