Files
yanxuan/website/_build_probe_data.py

275 lines
11 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
构建「择校决策」数据层(v3)
为什么换数据源
--------------
v2 的榜单基础是 core_hist_lines(考研365 复试线),抽样发现两个硬伤:
1. 大量院校复试线就等于国家线,线差中位数为 0,没有区分度,排不出难度;
2. 2022/2023 年国家线缺专硕单列线,管理类联考专业线差失真。
改用平台硕士点级数据(2026 届口径),它直接给"结果"而非"门槛":
ggyx_ssd_master 骨架:校 × 院系 × 专业(51,397 个硕士点,933 校)
ggyx_major_score_stats 拟录取最低分/最高分、进复试最低分、录取人数
ggyx_major_competition_proxy 竞争分 0-100、四级分档、录取率、外校调剂数
ggyx_major_yoy_metrics 2023-2026 逐年拟录取人数与最低分
ggyx_ssd_tag 择校标签(捡漏 / 过线就能上 / 压分 / 不保护一志愿 …)
口径
----
- 粒度 = 校 × 院系 × 专业(同校同专业多院系重复属正常,不合并,因为院系分数线确实不同)
- nlq_min / fs_min 为 0 时是"无数据占位",一律置 None,不参与排序与匹配
- 竞争分 score 与 grade 是平台测算值,界面上必须与官方分数区分标注
- 省份/2027 计划数来自 schools.json(按院校代码 join),缺失置空
输出
----
data/probe_index.json 专业索引 + 总览 + 热榜
data/probe/{code}.json 该专业下全部硕士点(按竞争分升序 = 越好考越靠前)
data/school_probe.json 院校索引
"""
import json
import os
import sqlite3
import collections
DB = r'D:/择校/数据库/zexiao_2027.db'
OUT = r'D:/择校/website/data'
PDIR = os.path.join(OUT, 'probe')
os.makedirs(PDIR, exist_ok=True)
# 标签压缩:只保留对择校决策有强信号的
TAGS = {
'捡漏': 'ln', '过线就能上': 'ok', '压分': 'yk', '不保护一志愿': 'np',
'有调剂': 'tj', '火爆硕士点': 'ht', '招生人数少': 'sm', '复试比大': 'fb',
'双非慎报': 'sb', '参考书多': 'bk', '专业课难度大': 'zk', '死亡硕士点': 'sw',
'专业课给分足': 'gf',
}
TAG_NAME = {v: k for k, v in TAGS.items()}
def lv_main(s):
"""'985,211,双一流' -> '985';'211,双一流' -> '211';'双非' -> '双非'"""
s = s or ''
if '985' in s:
return '985'
if '211' in s:
return '211'
if '双一流' in s:
return '双一流'
return '双非'
def pos(v):
"""0/None 视为无数据"""
try:
v = float(v)
except (TypeError, ValueError):
return None
return v if v > 0 else None
def main():
con = sqlite3.connect(DB)
con.row_factory = sqlite3.Row
schools = json.load(open(os.path.join(OUT, 'schools.json'), encoding='utf-8'))
by_code = {}
for s in schools:
c = s.get('code27')
if c and c not in by_code:
by_code[c] = s
sql = '''
select m.school_id, m.school_name, m.school_code, m.school_level, m.xy_name,
m.xy_id, m.is_quan,
m.major_id, m.major_code, m.major_name, m.major_one_name,
s.nlq_min, s.nlq_max, s.fs_min, s.nlq_cnt, s.fs_cnt,
p.score, p.grade, p.luqu_rate, p.not_full, p.wai_tj_num, p.hot_num,
y.nlq_2023, y.nlq_2024, y.nlq_2025, y.nlq_2026,
y.min_2023, y.min_2024, y.min_2025, y.min_2026
from ggyx_ssd_master m
left join ggyx_major_score_stats s
on m.school_id=s.school_id and m.xy_id=s.xy_id and m.major_id=s.major_id and m.is_quan=s.is_quan
left join ggyx_major_competition_proxy p
on m.school_id=p.school_id and m.xy_id=p.xy_id and m.major_id=p.major_id and m.is_quan=p.is_quan
left join ggyx_major_yoy_metrics y
on m.school_id=y.school_id and m.xy_id=y.xy_id and m.major_id=y.major_id and m.is_quan=y.is_quan
where m.major_code is not null
'''
rows = con.execute(sql).fetchall()
print('硕士点总行数:', len(rows))
tags = collections.defaultdict(set)
for r in con.execute('select school_id,xy_id,major_id,is_quan,tag_name from ggyx_ssd_tag'):
code = TAGS.get(r['tag_name'])
if code:
tags[(r['school_id'], r['xy_id'], r['major_id'], r['is_quan'])].add(code)
tree = collections.defaultdict(list)
for r in rows:
code = str(r['major_code']).strip()
if not code:
continue
sc = by_code.get(str(r['school_code'] or '').strip()) or {}
t = tags.get((r['school_id'], r['xy_id'], r['major_id'], r['is_quan']))
yoy = []
for y in (2023, 2024, 2025, 2026):
ln = pos(r['nlq_%d' % y])
mn = pos(r['min_%d' % y])
if ln is None and mn is None:
yoy.append(0)
else:
yoy.append([ln or 0, mn or 0])
item = {
'sid': r['school_id'],
'sch': r['school_name'],
'lv': lv_main(r['school_level']),
'xy': r['xy_name'] or '',
'pv': sc.get('prov') or '',
'img': pos(r['nlq_min']),
'imx': pos(r['nlq_max']),
'ifm': pos(r['fs_min']),
'ic': r['nlq_cnt'] or 0,
'sc': r['score'],
'gr': r['grade'],
'rt': r['luqu_rate'],
'tj': r['wai_tj_num'],
'd27': sc.get('d27') or 0,
'en': sc.get('enroll27') or 0,
'yo': yoy,
'tg': sorted(t) if t else [],
}
tree[code].append(item)
# 专业名(优先 majors.json,其次平台名)
majors = json.load(open(os.path.join(OUT, 'majors.json'), encoding='utf-8'))
mb = {m['code']: m for m in majors}
name_by_code = {}
for r in rows:
c = str(r['major_code']).strip()
if c and c not in name_by_code:
name_by_code[c] = r['major_name'] or ''
one_by_code = {}
for r in rows:
c = str(r['major_code']).strip()
if c and c not in one_by_code:
one_by_code[c] = r['major_one_name'] or ''
index = []
for code, items in tree.items():
m = mb.get(code)
name = (m['name'] if m else name_by_code.get(code)) or code
ml = (m['ml'] if m else '') or ''
# 有竞争分的按竞争分升序;无竞争分的排在后面按录取分升序
items.sort(key=lambda x: (x['sc'] is None, x['sc'] if x['sc'] is not None else 999,
x['img'] is None, x['img'] or 9999))
# 排序名次
for i, it in enumerate(items):
it['rk'] = i + 1
payload = {
'code': code,
'name': name,
'ml': ml or one_by_code.get(code, ''),
'one': one_by_code.get(code, ''),
'n': len(items),
'rows': items,
}
with open(os.path.join(PDIR, code + '.json'), 'w', encoding='utf-8') as f:
json.dump(payload, f, ensure_ascii=False, separators=(',', ':'))
withimg = [x for x in items if x['img']]
withsc = [x for x in items if x['sc'] is not None]
cold = sum(1 for x in items if x['gr'] == '偏冷')
pick = sum(1 for x in items if 'ln' in x['tg'] or 'ok' in x['tg'])
index.append({
'code': code, 'name': name, 'ml': ml or one_by_code.get(code, ''),
'n': len(items), 'ns': (m['n_school'] if m else 0),
'wsc': len(withsc), 'wimg': len(withimg),
'cold': cold, 'pick': pick,
'med': round(sum(x['sc'] for x in withsc) / len(withsc), 1) if withsc else None,
'lo': min((x['img'] for x in withimg), default=None),
'hi': max((x['img'] for x in withimg), default=None),
})
index.sort(key=lambda x: -x['n'])
overview = {
'n_school': con.execute('select count(distinct school_id) from ggyx_ssd_master').fetchone()[0],
'n_point': len(rows),
'n_code': len(index),
'n_scored': con.execute('select count(*) from ggyx_major_competition_proxy').fetchone()[0],
'n_admit': con.execute("select count(*) from ggyx_major_score_stats where nlq_min>0").fetchone()[0],
'n_tag': con.execute('select count(*) from ggyx_ssd_tag').fetchone()[0],
'grade': {r['grade']: r['c'] for r in con.execute(
'select grade,count(*) c from ggyx_major_competition_proxy group by grade')},
}
with open(os.path.join(OUT, 'probe_index.json'), 'w', encoding='utf-8') as f:
json.dump({'overview': overview, 'tags': TAG_NAME, 'majors': index}, f,
ensure_ascii=False, separators=(',', ':'))
# 院校索引(含名称/省份/层次/2027 计划,供首页搜索与院校页)
sch = collections.defaultdict(lambda: {'sc': [], 'n': 0, 'img': [], 'cold': 0,
'pick': 0, 'name': '', 'pv': '', 'lv': '', 'code': ''})
for code, items in tree.items():
for it in items:
d = sch[it['sid']]
d['n'] += 1
d['name'] = it['sch']
d['pv'] = it['pv'] or d['pv']
d['lv'] = it['lv'] or d['lv']
if it['sc'] is not None:
d['sc'].append(it['sc'])
if it['img']:
d['img'].append(it['img'])
if it['gr'] == '偏冷':
d['cold'] += 1
if 'ln' in it['tg'] or 'ok' in it['tg']:
d['pick'] += 1
out = []
for sid, d in sch.items():
info = by_code.get(d.get('code') or '') or {}
out.append({'sid': sid, 'n': d['n'], 'name': d['name'], 'pv': d['pv'], 'lv': d['lv'],
'sc': round(sum(d['sc']) / len(d['sc']), 1) if d['sc'] else None,
'img': min(d['img']) if d['img'] else None,
'cold': d['cold'], 'pick': d['pick'],
'd27': info.get('d27') or 0, 'en27': info.get('enroll27') or 0})
out.sort(key=lambda x: -x['n'])
with open(os.path.join(OUT, 'school_probe.json'), 'w', encoding='utf-8') as f:
json.dump({'n': len(out), 'schools': out}, f, ensure_ascii=False, separators=(',', ':'))
# 按院校分片:院校页展示"这所学校各专业好不好考"
SDIR = os.path.join(OUT, 'school')
os.makedirs(SDIR, exist_ok=True)
by_sid = collections.defaultdict(list)
for code, items in tree.items():
for it in items:
by_sid[it['sid']].append({
'code': code,
'name': (mb.get(code) or {}).get('name') or name_by_code.get(code) or code,
'ml': (mb.get(code) or {}).get('ml') or '',
'xy': it['xy'], 'sc': it['sc'], 'gr': it['gr'],
'img': it['img'], 'imx': it['imx'], 'ic': it['ic'],
'rt': it['rt'], 'rk': it['rk'], 'tg': it['tg'], 'yo': it['yo'],
})
meta = {d['sid']: d for d in out}
for sid, items in by_sid.items():
items.sort(key=lambda x: (x['sc'] is None, x['sc'] if x['sc'] is not None else 999))
m0 = meta.get(sid) or {}
with open(os.path.join(SDIR, str(sid) + '.json'), 'w', encoding='utf-8') as f:
json.dump({'sid': sid, 'name': m0.get('name', ''), 'pv': m0.get('pv', ''),
'lv': m0.get('lv', ''), 'n': len(items), 'rows': items}, f,
ensure_ascii=False, separators=(',', ':'))
print('院校分片:', len(by_sid))
print('专业码:', len(index))
print('总览:', overview)
print('TOP10:', [(x['name'], x['n'], x['wsc'], x['med']) for x in index[:10]])
tot = sum(os.path.getsize(os.path.join(PDIR, f)) for f in os.listdir(PDIR))
print('probe 体积 %.1f MB / %d 文件' % (tot / 1e6, len(os.listdir(PDIR))))
if __name__ == '__main__':
main()