Files
yanxuan/website/_build_rank_data.py

268 lines
11 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
导出择校决策数据层:国家线 + 院校专业历年复试线(线差)+ 院校索引
口径要点
--------
1. 分数线来源 core_hist_lines(2020-2024,源:考研365 各校分数线页),
一条记录 = 某校某专业某年的复试基本线。
2. 国家线来源 src_national_lines(2022-2026,研招网原文),按 codes 字段
精确匹配专业类(如 1253 会计 = 201 分),未命中时退回门类「其他学科专业」。
3. 线差 gap = 院校复试线 - 同年同专业类国家线(A 区)。这是跨年份可比的
难度指标,直接消除国家线波动。
4. 同校同专业同年可能存在多个学院多条线,取 MIN(total) 作为该校该专业的
最低门槛,并记录学院数 colleges。
5. 计划数 plan 直接取源表字段,填充率 78.2%,为空则置 None,不臆造。
输出
----
data/national.json 国家线(按年)
data/rank_index.json 专业码索引(供首页热榜/搜索)
data/rank/{code}.json 该专业下院校难度明细(按需加载)
data/school_lines.json 院校索引(全校线差、有线专业数)
"""
import json
import os
import sqlite3
import collections
DB = r'D:/择校/数据库/zexiao_2027.db'
OUT = r'D:/择校/website/data'
RANK_DIR = os.path.join(OUT, 'rank')
# 专业码前两位 -> 门类名(国家线 category_code 体系)
CAT = {
'01': '哲学', '02': '经济学', '03': '法学', '04': '教育学', '05': '文学',
'06': '历史学', '07': '理学', '08': '工学', '09': '农学', '10': '医学',
'11': '军事学', '12': '管理学', '13': '艺术学', '14': '交叉学科',
}
os.makedirs(RANK_DIR, exist_ok=True)
# 2022/2023 年国家线源缺失「专硕单列线」,只有门类合并线。
# 这些类别的专硕当年实际另划线(如 2022 会计 193 vs 管理学 353),
# 用门类线算线差会严重失真,故这两年一律不给线差,宁缺勿错。
SPLIT_2022_2023 = ('1251', '1252', '1253', '1254', '1255', '1256', '1257',
'0452', '0451', '0453', '1057')
def clean_name(s):
"""去掉 "(专业学位)" 之类前缀修饰,保留可读名。"""
if not s:
return ''
s = s.replace('(专业学位)', '').replace('(专业学位)', '')
return s.strip()
def main():
con = sqlite3.connect(DB)
con.row_factory = sqlite3.Row
# ---------- 1. 国家线 ----------
nat_rows = [dict(r) for r in con.execute(
'select year,category,category_code,subject,codes,a_total,a_single100,a_single101,policy '
'from src_national_lines where a_total is not null')]
national = collections.defaultdict(list)
for r in nat_rows:
codes = []
if r['codes']:
try:
codes = json.loads(r['codes'])
except Exception:
codes = []
national[str(r['year'])].append({
'cat': r['category_code'],
'name': r['category'],
'subj': r['subject'] or '',
'codes': codes,
'total': r['a_total'],
's100': r['a_single100'],
's101': r['a_single101'],
})
def nat_for(code, year):
"""按 专业类码(codes 精确) > 门类「其他/各学科专业」> 该门类首条 的顺序取国家线。
2022/2023 年对已单列但源缺失的专硕类别返回 None。"""
pool = national.get(str(year)) or []
if not pool:
return None
if str(year) in ('2022', '2023') and code[:4] in SPLIT_2022_2023:
return None
c4 = code[:4]
for it in pool:
if it['codes'] and c4 in it['codes']:
return it
cat = code[:2]
loose = [it for it in pool if it['cat'] == cat and not it['codes']]
for it in loose:
if '其他学科' in it['subj'] or '各学科' in it['subj']:
return it
if loose:
return loose[0]
return None
with open(os.path.join(OUT, 'national.json'), 'w', encoding='utf-8') as f:
json.dump({'years': sorted(national.keys()), 'by': national}, f,
ensure_ascii=False, separators=(',', ':'))
# ---------- 2. 院校库(去曾用名重复,按 id 建索引) ----------
schools = json.load(open(os.path.join(OUT, 'schools.json'), encoding='utf-8'))
sid_info = {}
for s in schools:
sid = s.get('id')
if sid is None or sid in sid_info:
continue
sid_info[sid] = {
'name': s['name'], 'prov': s.get('prov') or '', 'city': s.get('city') or '',
'level': s.get('level') or '双非', 'd27': s.get('d27') or 0,
'm27': s.get('m27') or 0, 'enroll27': s.get('enroll27') or 0,
}
majors = json.load(open(os.path.join(OUT, 'majors.json'), encoding='utf-8'))
maj_by_code = {m['code']: m for m in majors}
# ---------- 3. 分数线聚合 ----------
raw = con.execute('''
select school_id_ggyx as sid, school, code, name, year, total, plan, college
from core_hist_lines
where code is not null and length(code) >= 4 and total is not null and total > 0
and (data_flag is null)
''').fetchall()
# (code, sid, year) -> {line, colleges:set, plan}
agg = collections.defaultdict(lambda: {'line': None, 'colleges': set(), 'plan': None})
code_names = {}
for r in raw:
code = str(r['code'])
sid = r['sid']
y = int(r['year'])
k = (code, sid, y)
a = agg[k]
t = r['total']
a['line'] = t if a['line'] is None else min(a['line'], t)
a['colleges'].add(r['college'] or '')
if r['plan'] and r['plan'] > 0:
# 同校同专业同年多行时,plan 常是专业级数值重复出现,取 MAX 避免虚高
a['plan'] = r['plan'] if a['plan'] is None else max(a['plan'], r['plan'])
if code not in code_names and r['name']:
code_names[code] = clean_name(r['name'])
# code -> sid -> [year rows]
tree = collections.defaultdict(lambda: collections.defaultdict(list))
for (code, sid, y), a in agg.items():
nat = nat_for(code, y)
gap = None
nat_total = None
if nat and nat.get('total'):
nat_total = nat['total']
gap = a['line'] - nat_total
tree[code][sid].append({
'y': y, 'l': a['line'], 'n': nat_total, 'g': gap,
'p': a['plan'], 'c': len([c for c in a['colleges'] if c]),
})
# ---------- 4. 逐专业输出 ----------
index = []
for code, by_sid in tree.items():
rows = []
gaps_all = []
for sid, yrs in by_sid.items():
info = sid_info.get(sid)
if not info:
continue
yrs.sort(key=lambda x: x['y'])
# 近三年线差(越近权重越高:1,2,3)
gs = [x for x in yrs if x['g'] is not None]
g3 = None
if gs:
recent = gs[-3:]
w = list(range(1, len(recent) + 1))
g3 = round(sum(x['g'] * wi for x, wi in zip(recent, w)) / sum(w), 1)
gaps_all.append(g3)
last = ys = None
for x in yrs:
if x['l']:
last = x
plan = None
for x in yrs:
if x['p']:
plan = x['p']
rows.append({
'id': sid, 'name': info['name'], 'prov': info['prov'], 'lv': info['level'],
'ys': yrs, 'g3': g3, 'gy': len(gs), 'last': last['l'] if last else None,
'ly': last['y'] if last else None, 'plan': plan,
'd27': info['d27'], 'm27': info['m27'], 'en27': info['enroll27'],
})
if not rows:
continue
rows.sort(key=lambda r: (r['g3'] is None, -(r['g3'] or 0)))
m = maj_by_code.get(code)
name = (m['name'] if m else code_names.get(code)) or code
ml = m['ml'] if m else CAT.get(code[:2], '')
payload = {
'code': code, 'name': name, 'ml': ml,
'n': len(rows),
'rows': rows,
}
with open(os.path.join(RANK_DIR, code + '.json'), 'w', encoding='utf-8') as f:
json.dump(payload, f, ensure_ascii=False, separators=(',', ':'))
index.append({
'code': code, 'name': name, 'ml': ml, 'n': len(rows),
'ns': (m['n_school'] if m else 0),
'med': sorted(gaps_all)[len(gaps_all) // 2] if gaps_all else None,
'min': min(gaps_all) if gaps_all else None,
'max': max(gaps_all) if gaps_all else None,
})
index.sort(key=lambda x: -x['n'])
with open(os.path.join(OUT, 'rank_index.json'), 'w', encoding='utf-8') as f:
json.dump({'years': [2020, 2021, 2022, 2023, 2024],
'n_schools': len({r['sid'] for r in raw}),
'n_codes': len(index),
'majors': index}, f, ensure_ascii=False, separators=(',', ':'))
# ---------- 5. 院校索引 ----------
sch = {}
for code, by_sid in tree.items():
for sid, yrs in by_sid.items():
info = sid_info.get(sid)
if not info:
continue
d = sch.setdefault(sid, {'id': sid, **info, 'gaps': [], 'majors': []})
gs = [x['g'] for x in yrs if x['g'] is not None]
if gs:
d['gaps'].append(sum(gs) / len(gs))
d['majors'].append({'code': code, 'name': (maj_by_code.get(code) or {}).get('name') or code_names.get(code) or code,
'g': round(sum(gs) / len(gs), 1) if gs else None,
'last': [x['l'] for x in yrs if x['l']][-1] if gs else None})
out_sch = []
for sid, d in sch.items():
g = d['gaps']
d['majors'].sort(key=lambda x: (x['g'] is None, -(x['g'] or 0)))
out_sch.append({
'id': sid, 'name': d['name'], 'prov': d['prov'], 'level': d['level'],
'nm': len(d['majors']),
'g': round(sum(g) / len(g), 1) if g else None,
'd27': d['d27'], 'm27': d['m27'], 'en27': d['enroll27'],
})
out_sch.sort(key=lambda x: -x['nm'])
with open(os.path.join(OUT, 'school_lines.json'), 'w', encoding='utf-8') as f:
json.dump({'n': len(out_sch), 'schools': out_sch}, f, ensure_ascii=False, separators=(',', ':'))
# ---------- 诊断 ----------
print('专业码输出:', len(index))
print('院校有分数线:', len({r['sid'] for r in raw}))
print('有国家线的年份:', sorted(national.keys()))
print('TOP10 专业(校数):', [(x['name'], x['n'], x['med']) for x in index[:10]])
print('院校索引:', len(out_sch))
big = [x for x in index if x['n'] >= 20]
print('>=20 校的专业:', len(big))
total = sum(os.path.getsize(os.path.join(RANK_DIR, f)) for f in os.listdir(RANK_DIR))
print('rank 目录体积: %.1f MB / %d 文件' % (total / 1e6, len(os.listdir(RANK_DIR))))
if __name__ == '__main__':
main()