补上研招网目录里的官方推免数取数链并修缓存路径

yjfxs.do 的 ssjstmrs 是学院×方向级的「拟接收推免数」,当年入 chsi_2027_dir 时
只留了 enroll_str/enroll_num,把这列丢了。新增解析入库脚本落独立表
chsi_2027_tuimian_official(G5 方向级),不与公示口径 y2027_tuimian_2027 合并:
17 个重叠键上两口径没有一个相等(9 高 8 低),相加或替代都不可。
fetch_chsi.py 的缓存根原先是硬编码迁移前的 D:/zexiao_cache,直接跑会把已完成的
4728 个分片当空白重下,改成 --cache 可显式指定。
实测 ssjstmrs 的 0 有两种含义(70/155 校整列未填;东北大学三码官方全 0 而公示
给到 126/60/25 人),判据已登记 dim_field_semantics,本表只当计划上限用。
This commit is contained in:
Your Name committed 2026-11-02 16:10:21 +08:00
1 parent 8924c41c91
commit 38f51620dd
4 files changed
+355

No files matched your search

+13
View File
@@ -244,6 +244,7 @@ def run_shard(fc, dwdm, dwmc, yjxkdm):
def main():
global CACHE, RAW_MAJORS, RAW_DIRS, STATE, MANIFEST, ABORT_FILE
ap = argparse.ArgumentParser(description="研招网 2027 目录取数器(只读 GET,D 盘缓存)")
ap.add_argument("--root", default=r"C:\Users\12742\Desktop\择校", help="项目根,读院校清单用")
ap.add_argument("--yjxkdm", required=True, help="门类/一级学科码,逗号分隔,如 0854,0812")
@@ -254,8 +255,20 @@ def main():
ap.add_argument("--max-retry", type=int, default=3)
ap.add_argument("--max-requests", type=int, default=200, help="本次硬预算,到量即停并保留续跑点")
ap.add_argument("--rerun", action="store_true", help="忽略已完成分片,重跑")
ap.add_argument("--cache", default=CACHE,
help="缓存根。默认是迁移前的 D:/zexiao_cache/chsi_2027;"
"本项目已迁到 运行暂存/ 下,续跑时必须显式指过去,"
"否则已完成的分片会被当成空白重新下载")
a = ap.parse_args()
CACHE = a.cache.replace("\\", "/").rstrip("/")
RAW_MAJORS = CACHE + "/raw/majors"
RAW_DIRS = CACHE + "/raw/directions"
STATE = CACHE + "/state"
MANIFEST = CACHE + "/manifest.jsonl"
ABORT_FILE = CACHE + "/ABORT.json"
log("缓存根=%s" % CACHE)
ensure_dirs()
codes = [c.strip() for c in a.yjxkdm.split(",") if c.strip()]
@@ -0,0 +1,76 @@
# -*- coding: utf-8 -*-
"""给 0856 占比表加官方目录口径三列(公示口径与目录口径并排,不相加)。
占比表原有的是公示口径(院校名单实际接收),新表 chsi_2027_tuimian_official 是
目录口径(学校编计划时填的拟接收)。两者在重叠键上没有一个相等,
所以只能并排展示并标出方向行数与截断,供使用者判断该用哪个。
"""
import collections
import csv
import io
import re
import sqlite3
TBL = r'D:/择校/审计报告/0856_统考名额与推免占比_20261101.csv'
ZX = r'D:/择校/数据库/zexiao_2027.db'
ADD = ['官方目录拟接收推免', '官方未截断部分', '官方方向行数', '官方含截断',
'公示减官方目录']
def key(s):
s = re.sub(r'\s+', '', str(s or ''))
return s.replace('(', '(').replace(')', ')')
z = sqlite3.connect('file:' + ZX + '?mode=ro', uri=True)
off = collections.defaultdict(lambda: [0, 0, 0, 0])
for dwmc, zydm, t, tr in z.execute(
"select dwmc,zydm,tuimian,truncated_page1_only from chsi_2027_tuimian_official "
"where zydm like '0856%'"):
k = (key(dwmc), zydm)
off[k][0] += t or 0
if not tr:
off[k][1] += t or 0
off[k][2] += 1
off[k][3] = max(off[k][3], tr or 0)
nsch = z.execute("select count(distinct dwmc) from chsi_2027_tuimian_official "
"where zydm like '0856%'").fetchone()[0]
z.close()
rows = list(csv.DictReader(open(TBL, encoding='utf-8-sig')))
cols = list(rows[0].keys())
hit = 0
for r in rows:
k = (key(r['院校']), r['专业代码'])
o = off.get(k)
if o:
hit += 1
r['官方目录拟接收推免'] = o[0]
r['官方未截断部分'] = o[1]
r['官方方向行数'] = o[2]
r['官方含截断'] = '是' if o[3] else ''
r['公示减官方目录'] = int(r['推免已接收']) - o[0]
else:
for c in ADD:
r[c] = ''
r['公示减官方目录'] = ''
new_cols = cols[:5] + [c for c in ADD if c not in cols] + cols[5:]
with io.open(TBL, 'w', newline='', encoding='utf-8-sig') as f:
w = csv.DictWriter(f, fieldnames=new_cols, extrasaction='ignore')
w.writeheader()
w.writerows(rows)
L = ['=== 占比表接入目录口径 ===']
L.append('目录口径已覆盖 %d 校;占比表 %d 行里有 %d 行拿到官方数(其余是该校方向未取到)' % (
nsch, len(rows), hit))
d = [int(r['公示减官方目录']) for r in rows if r['公示减官方目录'] != '']
eq = sum(1 for x in d if x == 0)
L.append('两口径都有的 %d 行:相等 %d 行、公示更高 %d 行、公示更低 %d 行 → '
'目录口径不能替代公示口径,只做计划上限' % (len(d), eq, sum(1 for x in d if x > 0),
sum(1 for x in d if x < 0)))
tr = sum(1 for r in rows if r['官方含截断'] == '是')
L.append('官方数受第1页10条截断影响的行=%d(这些行的官方数是下界,不是全量)' % tr)
L.append('列顺序=%s' % ','.join(new_cols))
open(r'D:/择校/运行暂存/agent_ingest_20261101/official_cols_20261102.txt', 'w',
encoding='utf-8').write('\n'.join(L) + '\n')
print('\n'.join(L))
@@ -0,0 +1,95 @@
# -*- coding: utf-8 -*-
"""官方目录推免数 vs 院校公示推免数 交叉核对(只读,分列不合并)。
两张表口径不同:
chsi_2027_tuimian_official = 学校编 2027 目录时填的「拟接收」推免数;
y2027_tuimian_2027/_major = 推免接收名单公示出来的实际人数(时点更晚)。
差值方向本身就是信息(官方留了余量 / 公示超出计划),但不能相加或取平均,
所以这里只输出对照与分档,不动任何一张表的值。
"""
import collections
import csv
import io
import re
import sqlite3
ZX = r'D:/择校/数据库/zexiao_2027.db'
OUT = r'D:/择校/审计报告/0856_推免两口径对照_20261102.csv'
LOG = r'D:/择校/运行暂存/agent_ingest_20261101/tuimian_two_calibers.txt'
def key(s):
"""校名归一:全角括号转半角、去空白(研招网用半角,公示常写全角)。"""
s = re.sub(r'\s+', '', str(s or ''))
return s.replace('(', '(').replace(')', ')').replace(':', ':')
z = sqlite3.connect('file:' + ZX + '?mode=ro', uri=True)
off = collections.defaultdict(lambda: [0, 0, 0]) # (校,码) -> [全部, 未截断, 方向行数]
for dwmc, zydm, t, tr in z.execute(
"select dwmc,zydm,tuimian,truncated_page1_only from chsi_2027_tuimian_official "
"where zydm like '0856%'"):
k = (key(dwmc), zydm)
off[k][0] += t or 0
if not tr:
off[k][1] += t or 0
off[k][2] += 1
pub = collections.defaultdict(int)
pub_detail = collections.defaultdict(list)
for t, vc in [('y2027_tuimian_2027', 'value'), ('y2027_tuimian_major', 'received_recommend_count')]:
for s, m, v in z.execute("select school, major_code, %s from %s" % (vc, t)):
if re.match(r'^\d+$', str(v).strip()) and str(m).startswith('0856'):
pub[(key(s), str(m).strip())] += int(v)
pub_detail[(key(s), str(m).strip())].append(t)
z.close()
L = []
L.append('=== 0856 推免两口径对照 ===')
L.append('官方目录口径键=%d(覆盖 %d 校) 公示口径键=%d(覆盖 %d 校)' % (
len(off), len({k[0] for k in off}), len(pub), len({k[0] for k in pub})))
both = sorted(set(off) & set(pub))
only_off = sorted(set(off) - set(pub))
only_pub = sorted(set(pub) - set(off))
L.append('两口径都有=%d 键 只有官方=%d 键 只有公示=%d 键' % (len(both), len(only_off), len(only_pub)))
band = collections.Counter()
rows = []
for k in sorted(set(off) | set(pub)):
o = off.get(k, [0, 0, 0])
p = pub.get(k, 0)
if k in both:
d = p - o[0]
if d == 0:
band['相等'] += 1
elif d > 0:
band['公示>官方'] += 1
else:
band['公示<官方'] += 1
rows.append({'院校': k[0], '专业代码': k[1],
'官方拟接收合计': o[0] if k in off else '',
'官方未截断部分': o[1] if k in off else '',
'官方方向行数': o[2] if k in off else '',
'公示实际接收': p if k in pub else '',
'差(公示-官方)': (p - o[0]) if k in both else '',
'覆盖情况': ('两口径都有' if k in both else
('只有官方目录' if k in off else '只有公示'))})
L.append('两口径都有键上的关系: %s' % band.most_common())
L.append('')
L.append('差值最大的 12 键(公示 − 官方):')
for r in sorted([x for x in rows if x['差(公示-官方)'] != ''],
key=lambda x: -abs(x['差(公示-官方)']))[:12]:
L.append(' %-14s %s 官方=%-5s 公示=%-5s 差=%-6s 官方方向行=%s' % (
r['院校'], r['专业代码'], r['官方拟接收合计'], r['公示实际接收'],
r['差(公示-官方)'], r['官方方向行数']))
L.append('')
L.append('只有官方没有公示的校(前 12,是后续公示名单的靶子):')
for k in only_off[:12]:
L.append(' %-16s %s 官方拟接收=%d' % (k[0][:16], k[1], off[k][0]))
with io.open(OUT, 'w', newline='', encoding='utf-8-sig') as f:
w = csv.DictWriter(f, fieldnames=list(rows[0].keys()))
w.writeheader()
w.writerows(rows)
L.append('')
L.append('对照表写入 %s(%d 行)' % (OUT, len(rows)))
open(LOG, 'w', encoding='utf-8').write('\n'.join(L) + '\n')
print('\n'.join(L))
@@ -0,0 +1,171 @@
# -*- coding: utf-8 -*-
"""解析研招网 2027 目录方向响应里的官方推免数,落独立新表。
为什么单独一张表而不并进 y2027_tuimian_2027:
- 这张是**目录口径**(学校编计划时填的拟接收推免数,ssjstmrs,方向级);
- y2027_tuimian_2027 是**公示口径**(推免接收名单实际公示的人数);
- 两者时间点和含义都不同(清华的备注就写着「招生人数后续可能调整」),
合并会让交付层的数无法回溯,所以分列两表,交叉核对只做差值分布。
自洽性实测:恒有 nzsrs = ssjstmrs + nzsrsstr 里的「不含推免」数
(清华深研院 115 = 51 + 64、材料学院 20 = 16 + 4),所以 ssjstmrs 是方向级值,
不是学院级重复值 —— 学院级要自己按 yxsdm 汇总。
"""
import collections
import glob
import json
import os
import re
import sqlite3
import time
ZX = r'D:/择校/数据库/zexiao_2027.db'
RAW = r'D:/择校/运行暂存/zexiao_cache/chsi_2027/raw/directions'
TAB = 'chsi_2027_tuimian_official'
DIAG = r'D:/择校/运行暂存/agent_ingest_20261101/official_tuimian_diag.txt'
REFUSE = re.compile(r'访问太频繁|请登录|参数错误|频繁|验证|无权|失败')
CODE = re.compile(r'^((?:0[1-9]|1[0-4])\d{4}|\d{4}[A-Z]\d{1,2})$')
def norm(s):
return re.sub(r'\s+', '', str(s or ''))
def parse_excl(s):
"""nzsrsstr 有三种粒度前缀,返回 (口径, 不含推免数, 原文)。"""
s = str(s or '')
m = re.search(r'(专业|一级学科|院系所)\s*[::]\s*(\d+)', s)
if m:
return m.group(1), int(m.group(2)), s
m = re.search(r'研究方向\s*[::]\s*(\d+)', s)
if m:
return '研究方向', int(m.group(1)), s
return '', None, s
def main():
files = sorted(glob.glob(os.path.join(RAW, '*_0856*.json')))
rows, bad_refusal, n_trunc, n_item = [], 0, 0, 0
seen = set()
for p in files:
try:
d = json.load(open(p, encoding='utf-8'))
except Exception:
continue
m = d.get('msg')
if isinstance(m, str):
if REFUSE.search(m):
bad_refusal += 1
continue
if not isinstance(m, dict):
continue
lst = m.get('list') or []
tc = m.get('totalCount') or 0
if tc > len(lst):
n_trunc += 1
for it in lst:
if not isinstance(it, dict):
continue
zydm = norm(it.get('zydm'))
if not CODE.match(zydm):
continue
scope, excl, raw = parse_excl(it.get('nzsrsstr'))
ssj = it.get('ssjstmrs')
nz = it.get('nzsrs')
try:
ssj = int(str(ssj))
except Exception:
ssj = None
try:
nz = int(str(nz))
except Exception:
nz = None
k = (norm(it.get('dwdm')), zydm, norm(it.get('yxsdm')), norm(it.get('yjfxdm')),
norm(it.get('xxfs')))
if k in seen:
continue
seen.add(k)
n_item += 1
rows.append(dict(
dwdm=k[0], dwmc=norm(it.get('dwmc')), zydm=zydm, zymc=norm(it.get('zymc')),
yxsdm=k[2], yxsmc=norm(it.get('yxsmc')), yjfxdm=k[3],
yjfxmc=norm(it.get('yjfxmc')), xxfs=k[4], xxfsmc=norm(it.get('xxfsmc')),
tuimian=ssj, nzsrs=nz, excl=excl, scope=scope, raw=raw,
truncated=1 if tc > len(lst) else 0, zybz=norm(it.get('zybz'))[:200],
source_file=os.path.basename(p),
api='zsml/rs/yjfxs.do'))
z = sqlite3.connect(ZX, timeout=600)
z.execute('PRAGMA busy_timeout=600000')
before = z.execute('select count(*) from %s' % TAB).fetchone()[0] if z.execute(
"select 1 from sqlite_master where name=?", (TAB,)).fetchone() else None
if before is None:
z.execute('''CREATE TABLE %s (
"dwdm" TEXT,"dwmc" TEXT,"zydm" TEXT,"zymc" TEXT,"yxsdm" TEXT,"yxsmc" TEXT,
"yjfxdm" TEXT,"yjfxmc" TEXT,"xxfs" TEXT,"xxfsmc" TEXT,
"tuimian" INTEGER,"nzsrs" INTEGER,"excl_num" INTEGER,"quota_scope" TEXT,
"nzsrsstr" TEXT,"truncated_page1_only" INTEGER,"zybz" TEXT,
"source_file" TEXT,"api" TEXT,"_ingested_at" TEXT,
PRIMARY KEY ("dwdm","zydm","yxsdm","yjfxdm","xxfs"))''' % TAB)
now = time.strftime('%Y-%m-%dT%H:%M:%S')
COLS = ['dwdm', 'dwmc', 'zydm', 'zymc', 'yxsdm', 'yxsmc', 'yjfxdm', 'yjfxmc', 'xxfs',
'xxfsmc', 'tuimian', 'nzsrs', 'excl_num', 'quota_scope', 'nzsrsstr',
'truncated_page1_only', 'zybz', 'source_file', 'api', '_ingested_at']
ins = 'INSERT OR REPLACE INTO %s ("%s") VALUES (%s)' % (
TAB, '","'.join(COLS), ','.join('?' * len(COLS)))
n = 0
for r in rows:
z.execute(ins, (r['dwdm'], r['dwmc'], r['zydm'], r['zymc'], r['yxsdm'], r['yxsmc'],
r['yjfxdm'], r['yjfxmc'], r['xxfs'], r['xxfsmc'], r['tuimian'],
r['nzsrs'], r['excl'], r['scope'], r['raw'], r['truncated'],
r['zybz'], r['source_file'], r['api'], now))
n += 1
tot = z.execute('select count(*) from %s' % TAB).fetchone()[0]
sch = z.execute('select count(distinct dwdm) from %s' % TAB).fetchone()[0]
has = z.execute('select count(*) from %s where tuimian is not null' % TAB).fetchone()[0]
pos = z.execute('select count(*) from %s where coalesce(tuimian,0)>0' % TAB).fetchone()[0]
s2 = z.execute("select count(*), sum(tuimian) from %s where tuimian>0 and "
"truncated_page1_only=0" % TAB).fetchone()
self_ok = z.execute('select count(*) from %s where nzsrs is not null and excl_num is not null '
'and nzsrs = tuimian + excl_num' % TAB).fetchone()[0]
self_bad = z.execute('select count(*) from %s where nzsrs is not null and excl_num is not null '
'and nzsrs <> tuimian + excl_num' % TAB).fetchone()[0]
meta = ('研招网 zsml/rs/yjfxs.do 2027 硕士目录',
'官方目录口径的拟接收推免数 ssjstmrs;与公示口径 y2027_tuimian_2027 分列,'
'truncated_page1_only=1 表示该专业方向数超第1页10条未取全')
if z.execute('select 1 from dim_table_meta where table_name=?', (TAB,)).fetchone():
z.execute('update dim_table_meta set rows=?, grain=?, year_scope=?, school_col=?, '
'major_col=?, source=?, purpose=?, updated_at=? where table_name=?',
(tot, 'G5 方向级(学院×研究方向)', '2027', 'dwmc', 'zydm',
meta[0], meta[1], now, TAB))
else:
z.execute('INSERT INTO dim_table_meta VALUES (?,?,?,?,?,?,?,?,?,?,?)',
(TAB, tot, 'G5 方向级(学院×研究方向)', '2027', 'dwmc', 'zydm', 0, 0,
meta[0], meta[1], now))
z.execute('INSERT INTO changelog VALUES (?,?,?,?,?,?)',
(time.strftime('%Y-%m-%d %H:%M:%S'), 'chsi_tuimian_official_20261102', TAB,
'insert', n,
'解析 %d 份 0856 方向响应写入 %d 行;拒绝响应 %d 份、第1页截断 %d 份;'
'自洽校验 nzsrs=tuimian+excl 命中 %d / 不符 %d'
% (len(files), n, bad_refusal, n_trunc, self_ok, self_bad)))
z.commit()
L = ['=== 官方目录推免数入库 %s ===' % time.strftime('%Y-%m-%d %H:%M')]
L.append('响应文件=%d 解析条目=%d 写入=%d 表内合计=%d 覆盖校=%d' % (
len(files), n_item, n, tot, sch))
L.append('tuimian 非空=%d 其中 >0 =%d' % (has, pos))
L.append('未截断且 tuimian>0:行=%d 人数合计=%s' % (s2[0], s2[1]))
L.append('自洽校验 nzsrs = tuimian + excl_num:命中=%d 不符=%d(不符多因方向行给的是专业级名额)' % (
self_ok, self_bad))
L.append('第1页截断文件=%d 拒绝响应=%d' % (n_trunc, bad_refusal))
L.append('')
L.append('按校看官方推免数(未截断的方向行汇总,前 14):')
q = ("select dwmc, sum(tuimian) t, count(distinct zydm) c from %s "
"where truncated_page1_only=0 and coalesce(tuimian,0)>0 group by 1 order by t desc limit 14" % TAB)
for r in z.execute(q):
L.append(' %-16s 官方拟接收推免=%-6d 涉及码=%d' % (r[0][:16], r[1], r[2]))
z.close()
open(DIAG, 'w', encoding='utf-8').write('\n'.join(L) + '\n')
print('\n'.join(L))
if __name__ == '__main__':
main()