268 lines
11 KiB
Python
268 lines
11 KiB
Python
#!/usr/bin/env python
|
||
# -*- coding: utf-8 -*-
|
||
"""
|
||
导出择校决策数据层:国家线 + 院校专业历年复试线(线差)+ 院校索引
|
||
|
||
口径要点
|
||
--------
|
||
1. 分数线来源 core_hist_lines(2020-2024,源:考研365 各校分数线页),
|
||
一条记录 = 某校某专业某年的复试基本线。
|
||
2. 国家线来源 src_national_lines(2022-2026,研招网原文),按 codes 字段
|
||
精确匹配专业类(如 1253 会计 = 201 分),未命中时退回门类「其他学科专业」。
|
||
3. 线差 gap = 院校复试线 - 同年同专业类国家线(A 区)。这是跨年份可比的
|
||
难度指标,直接消除国家线波动。
|
||
4. 同校同专业同年可能存在多个学院多条线,取 MIN(total) 作为该校该专业的
|
||
最低门槛,并记录学院数 colleges。
|
||
5. 计划数 plan 直接取源表字段,填充率 78.2%,为空则置 None,不臆造。
|
||
|
||
输出
|
||
----
|
||
data/national.json 国家线(按年)
|
||
data/rank_index.json 专业码索引(供首页热榜/搜索)
|
||
data/rank/{code}.json 该专业下院校难度明细(按需加载)
|
||
data/school_lines.json 院校索引(全校线差、有线专业数)
|
||
"""
|
||
import json
|
||
import os
|
||
import sqlite3
|
||
import collections
|
||
|
||
DB = r'D:/择校/数据库/zexiao_2027.db'
|
||
OUT = r'D:/择校/website/data'
|
||
RANK_DIR = os.path.join(OUT, 'rank')
|
||
|
||
# 专业码前两位 -> 门类名(国家线 category_code 体系)
|
||
CAT = {
|
||
'01': '哲学', '02': '经济学', '03': '法学', '04': '教育学', '05': '文学',
|
||
'06': '历史学', '07': '理学', '08': '工学', '09': '农学', '10': '医学',
|
||
'11': '军事学', '12': '管理学', '13': '艺术学', '14': '交叉学科',
|
||
}
|
||
|
||
os.makedirs(RANK_DIR, exist_ok=True)
|
||
|
||
# 2022/2023 年国家线源缺失「专硕单列线」,只有门类合并线。
|
||
# 这些类别的专硕当年实际另划线(如 2022 会计 193 vs 管理学 353),
|
||
# 用门类线算线差会严重失真,故这两年一律不给线差,宁缺勿错。
|
||
SPLIT_2022_2023 = ('1251', '1252', '1253', '1254', '1255', '1256', '1257',
|
||
'0452', '0451', '0453', '1057')
|
||
|
||
|
||
def clean_name(s):
|
||
"""去掉 "(专业学位)" 之类前缀修饰,保留可读名。"""
|
||
if not s:
|
||
return ''
|
||
s = s.replace('(专业学位)', '').replace('(专业学位)', '')
|
||
return s.strip()
|
||
|
||
|
||
def main():
|
||
con = sqlite3.connect(DB)
|
||
con.row_factory = sqlite3.Row
|
||
|
||
# ---------- 1. 国家线 ----------
|
||
nat_rows = [dict(r) for r in con.execute(
|
||
'select year,category,category_code,subject,codes,a_total,a_single100,a_single101,policy '
|
||
'from src_national_lines where a_total is not null')]
|
||
national = collections.defaultdict(list)
|
||
for r in nat_rows:
|
||
codes = []
|
||
if r['codes']:
|
||
try:
|
||
codes = json.loads(r['codes'])
|
||
except Exception:
|
||
codes = []
|
||
national[str(r['year'])].append({
|
||
'cat': r['category_code'],
|
||
'name': r['category'],
|
||
'subj': r['subject'] or '',
|
||
'codes': codes,
|
||
'total': r['a_total'],
|
||
's100': r['a_single100'],
|
||
's101': r['a_single101'],
|
||
})
|
||
|
||
def nat_for(code, year):
|
||
"""按 专业类码(codes 精确) > 门类「其他/各学科专业」> 该门类首条 的顺序取国家线。
|
||
2022/2023 年对已单列但源缺失的专硕类别返回 None。"""
|
||
pool = national.get(str(year)) or []
|
||
if not pool:
|
||
return None
|
||
if str(year) in ('2022', '2023') and code[:4] in SPLIT_2022_2023:
|
||
return None
|
||
c4 = code[:4]
|
||
for it in pool:
|
||
if it['codes'] and c4 in it['codes']:
|
||
return it
|
||
cat = code[:2]
|
||
loose = [it for it in pool if it['cat'] == cat and not it['codes']]
|
||
for it in loose:
|
||
if '其他学科' in it['subj'] or '各学科' in it['subj']:
|
||
return it
|
||
if loose:
|
||
return loose[0]
|
||
return None
|
||
|
||
with open(os.path.join(OUT, 'national.json'), 'w', encoding='utf-8') as f:
|
||
json.dump({'years': sorted(national.keys()), 'by': national}, f,
|
||
ensure_ascii=False, separators=(',', ':'))
|
||
|
||
# ---------- 2. 院校库(去曾用名重复,按 id 建索引) ----------
|
||
schools = json.load(open(os.path.join(OUT, 'schools.json'), encoding='utf-8'))
|
||
sid_info = {}
|
||
for s in schools:
|
||
sid = s.get('id')
|
||
if sid is None or sid in sid_info:
|
||
continue
|
||
sid_info[sid] = {
|
||
'name': s['name'], 'prov': s.get('prov') or '', 'city': s.get('city') or '',
|
||
'level': s.get('level') or '双非', 'd27': s.get('d27') or 0,
|
||
'm27': s.get('m27') or 0, 'enroll27': s.get('enroll27') or 0,
|
||
}
|
||
|
||
majors = json.load(open(os.path.join(OUT, 'majors.json'), encoding='utf-8'))
|
||
maj_by_code = {m['code']: m for m in majors}
|
||
|
||
# ---------- 3. 分数线聚合 ----------
|
||
raw = con.execute('''
|
||
select school_id_ggyx as sid, school, code, name, year, total, plan, college
|
||
from core_hist_lines
|
||
where code is not null and length(code) >= 4 and total is not null and total > 0
|
||
and (data_flag is null)
|
||
''').fetchall()
|
||
|
||
# (code, sid, year) -> {line, colleges:set, plan}
|
||
agg = collections.defaultdict(lambda: {'line': None, 'colleges': set(), 'plan': None})
|
||
code_names = {}
|
||
for r in raw:
|
||
code = str(r['code'])
|
||
sid = r['sid']
|
||
y = int(r['year'])
|
||
k = (code, sid, y)
|
||
a = agg[k]
|
||
t = r['total']
|
||
a['line'] = t if a['line'] is None else min(a['line'], t)
|
||
a['colleges'].add(r['college'] or '')
|
||
if r['plan'] and r['plan'] > 0:
|
||
# 同校同专业同年多行时,plan 常是专业级数值重复出现,取 MAX 避免虚高
|
||
a['plan'] = r['plan'] if a['plan'] is None else max(a['plan'], r['plan'])
|
||
if code not in code_names and r['name']:
|
||
code_names[code] = clean_name(r['name'])
|
||
|
||
# code -> sid -> [year rows]
|
||
tree = collections.defaultdict(lambda: collections.defaultdict(list))
|
||
for (code, sid, y), a in agg.items():
|
||
nat = nat_for(code, y)
|
||
gap = None
|
||
nat_total = None
|
||
if nat and nat.get('total'):
|
||
nat_total = nat['total']
|
||
gap = a['line'] - nat_total
|
||
tree[code][sid].append({
|
||
'y': y, 'l': a['line'], 'n': nat_total, 'g': gap,
|
||
'p': a['plan'], 'c': len([c for c in a['colleges'] if c]),
|
||
})
|
||
|
||
# ---------- 4. 逐专业输出 ----------
|
||
index = []
|
||
for code, by_sid in tree.items():
|
||
rows = []
|
||
gaps_all = []
|
||
for sid, yrs in by_sid.items():
|
||
info = sid_info.get(sid)
|
||
if not info:
|
||
continue
|
||
yrs.sort(key=lambda x: x['y'])
|
||
# 近三年线差(越近权重越高:1,2,3)
|
||
gs = [x for x in yrs if x['g'] is not None]
|
||
g3 = None
|
||
if gs:
|
||
recent = gs[-3:]
|
||
w = list(range(1, len(recent) + 1))
|
||
g3 = round(sum(x['g'] * wi for x, wi in zip(recent, w)) / sum(w), 1)
|
||
gaps_all.append(g3)
|
||
last = ys = None
|
||
for x in yrs:
|
||
if x['l']:
|
||
last = x
|
||
plan = None
|
||
for x in yrs:
|
||
if x['p']:
|
||
plan = x['p']
|
||
rows.append({
|
||
'id': sid, 'name': info['name'], 'prov': info['prov'], 'lv': info['level'],
|
||
'ys': yrs, 'g3': g3, 'gy': len(gs), 'last': last['l'] if last else None,
|
||
'ly': last['y'] if last else None, 'plan': plan,
|
||
'd27': info['d27'], 'm27': info['m27'], 'en27': info['enroll27'],
|
||
})
|
||
if not rows:
|
||
continue
|
||
rows.sort(key=lambda r: (r['g3'] is None, -(r['g3'] or 0)))
|
||
m = maj_by_code.get(code)
|
||
name = (m['name'] if m else code_names.get(code)) or code
|
||
ml = m['ml'] if m else CAT.get(code[:2], '')
|
||
payload = {
|
||
'code': code, 'name': name, 'ml': ml,
|
||
'n': len(rows),
|
||
'rows': rows,
|
||
}
|
||
with open(os.path.join(RANK_DIR, code + '.json'), 'w', encoding='utf-8') as f:
|
||
json.dump(payload, f, ensure_ascii=False, separators=(',', ':'))
|
||
|
||
index.append({
|
||
'code': code, 'name': name, 'ml': ml, 'n': len(rows),
|
||
'ns': (m['n_school'] if m else 0),
|
||
'med': sorted(gaps_all)[len(gaps_all) // 2] if gaps_all else None,
|
||
'min': min(gaps_all) if gaps_all else None,
|
||
'max': max(gaps_all) if gaps_all else None,
|
||
})
|
||
|
||
index.sort(key=lambda x: -x['n'])
|
||
with open(os.path.join(OUT, 'rank_index.json'), 'w', encoding='utf-8') as f:
|
||
json.dump({'years': [2020, 2021, 2022, 2023, 2024],
|
||
'n_schools': len({r['sid'] for r in raw}),
|
||
'n_codes': len(index),
|
||
'majors': index}, f, ensure_ascii=False, separators=(',', ':'))
|
||
|
||
# ---------- 5. 院校索引 ----------
|
||
sch = {}
|
||
for code, by_sid in tree.items():
|
||
for sid, yrs in by_sid.items():
|
||
info = sid_info.get(sid)
|
||
if not info:
|
||
continue
|
||
d = sch.setdefault(sid, {'id': sid, **info, 'gaps': [], 'majors': []})
|
||
gs = [x['g'] for x in yrs if x['g'] is not None]
|
||
if gs:
|
||
d['gaps'].append(sum(gs) / len(gs))
|
||
d['majors'].append({'code': code, 'name': (maj_by_code.get(code) or {}).get('name') or code_names.get(code) or code,
|
||
'g': round(sum(gs) / len(gs), 1) if gs else None,
|
||
'last': [x['l'] for x in yrs if x['l']][-1] if gs else None})
|
||
out_sch = []
|
||
for sid, d in sch.items():
|
||
g = d['gaps']
|
||
d['majors'].sort(key=lambda x: (x['g'] is None, -(x['g'] or 0)))
|
||
out_sch.append({
|
||
'id': sid, 'name': d['name'], 'prov': d['prov'], 'level': d['level'],
|
||
'nm': len(d['majors']),
|
||
'g': round(sum(g) / len(g), 1) if g else None,
|
||
'd27': d['d27'], 'm27': d['m27'], 'en27': d['enroll27'],
|
||
})
|
||
out_sch.sort(key=lambda x: -x['nm'])
|
||
with open(os.path.join(OUT, 'school_lines.json'), 'w', encoding='utf-8') as f:
|
||
json.dump({'n': len(out_sch), 'schools': out_sch}, f, ensure_ascii=False, separators=(',', ':'))
|
||
|
||
# ---------- 诊断 ----------
|
||
print('专业码输出:', len(index))
|
||
print('院校有分数线:', len({r['sid'] for r in raw}))
|
||
print('有国家线的年份:', sorted(national.keys()))
|
||
print('TOP10 专业(校数):', [(x['name'], x['n'], x['med']) for x in index[:10]])
|
||
print('院校索引:', len(out_sch))
|
||
big = [x for x in index if x['n'] >= 20]
|
||
print('>=20 校的专业:', len(big))
|
||
total = sum(os.path.getsize(os.path.join(RANK_DIR, f)) for f in os.listdir(RANK_DIR))
|
||
print('rank 目录体积: %.1f MB / %d 文件' % (total / 1e6, len(os.listdir(RANK_DIR))))
|
||
|
||
|
||
if __name__ == '__main__':
|
||
main()
|