补上研招网目录里的官方推免数取数链并修缓存路径
yjfxs.do 的 ssjstmrs 是学院×方向级的「拟接收推免数」,当年入 chsi_2027_dir 时 只留了 enroll_str/enroll_num,把这列丢了。新增解析入库脚本落独立表 chsi_2027_tuimian_official(G5 方向级),不与公示口径 y2027_tuimian_2027 合并: 17 个重叠键上两口径没有一个相等(9 高 8 低),相加或替代都不可。 fetch_chsi.py 的缓存根原先是硬编码迁移前的 D:/zexiao_cache,直接跑会把已完成的 4728 个分片当空白重下,改成 --cache 可显式指定。 实测 ssjstmrs 的 0 有两种含义(70/155 校整列未填;东北大学三码官方全 0 而公示 给到 126/60/25 人),判据已登记 dim_field_semantics,本表只当计划上限用。
This commit is contained in:
1 parent
8924c41c91
commit
38f51620dd
4 files changed
+355
No files matched your search
@@ -244,6 +244,7 @@ def run_shard(fc, dwdm, dwmc, yjxkdm):
|
||||
|
||||
|
||||
def main():
|
||||
global CACHE, RAW_MAJORS, RAW_DIRS, STATE, MANIFEST, ABORT_FILE
|
||||
ap = argparse.ArgumentParser(description="研招网 2027 目录取数器(只读 GET,D 盘缓存)")
|
||||
ap.add_argument("--root", default=r"C:\Users\12742\Desktop\择校", help="项目根,读院校清单用")
|
||||
ap.add_argument("--yjxkdm", required=True, help="门类/一级学科码,逗号分隔,如 0854,0812")
|
||||
@@ -254,8 +255,20 @@ def main():
|
||||
ap.add_argument("--max-retry", type=int, default=3)
|
||||
ap.add_argument("--max-requests", type=int, default=200, help="本次硬预算,到量即停并保留续跑点")
|
||||
ap.add_argument("--rerun", action="store_true", help="忽略已完成分片,重跑")
|
||||
ap.add_argument("--cache", default=CACHE,
|
||||
help="缓存根。默认是迁移前的 D:/zexiao_cache/chsi_2027;"
|
||||
"本项目已迁到 运行暂存/ 下,续跑时必须显式指过去,"
|
||||
"否则已完成的分片会被当成空白重新下载")
|
||||
a = ap.parse_args()
|
||||
|
||||
CACHE = a.cache.replace("\\", "/").rstrip("/")
|
||||
RAW_MAJORS = CACHE + "/raw/majors"
|
||||
RAW_DIRS = CACHE + "/raw/directions"
|
||||
STATE = CACHE + "/state"
|
||||
MANIFEST = CACHE + "/manifest.jsonl"
|
||||
ABORT_FILE = CACHE + "/ABORT.json"
|
||||
log("缓存根=%s" % CACHE)
|
||||
|
||||
ensure_dirs()
|
||||
codes = [c.strip() for c in a.yjxkdm.split(",") if c.strip()]
|
||||
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""给 0856 占比表加官方目录口径三列(公示口径与目录口径并排,不相加)。
|
||||
|
||||
占比表原有的是公示口径(院校名单实际接收),新表 chsi_2027_tuimian_official 是
|
||||
目录口径(学校编计划时填的拟接收)。两者在重叠键上没有一个相等,
|
||||
所以只能并排展示并标出方向行数与截断,供使用者判断该用哪个。
|
||||
"""
|
||||
import collections
|
||||
import csv
|
||||
import io
|
||||
import re
|
||||
import sqlite3
|
||||
|
||||
TBL = r'D:/择校/审计报告/0856_统考名额与推免占比_20261101.csv'
|
||||
ZX = r'D:/择校/数据库/zexiao_2027.db'
|
||||
ADD = ['官方目录拟接收推免', '官方未截断部分', '官方方向行数', '官方含截断',
|
||||
'公示减官方目录']
|
||||
|
||||
|
||||
def key(s):
|
||||
s = re.sub(r'\s+', '', str(s or ''))
|
||||
return s.replace('(', '(').replace(')', ')')
|
||||
|
||||
|
||||
z = sqlite3.connect('file:' + ZX + '?mode=ro', uri=True)
|
||||
off = collections.defaultdict(lambda: [0, 0, 0, 0])
|
||||
for dwmc, zydm, t, tr in z.execute(
|
||||
"select dwmc,zydm,tuimian,truncated_page1_only from chsi_2027_tuimian_official "
|
||||
"where zydm like '0856%'"):
|
||||
k = (key(dwmc), zydm)
|
||||
off[k][0] += t or 0
|
||||
if not tr:
|
||||
off[k][1] += t or 0
|
||||
off[k][2] += 1
|
||||
off[k][3] = max(off[k][3], tr or 0)
|
||||
nsch = z.execute("select count(distinct dwmc) from chsi_2027_tuimian_official "
|
||||
"where zydm like '0856%'").fetchone()[0]
|
||||
z.close()
|
||||
|
||||
rows = list(csv.DictReader(open(TBL, encoding='utf-8-sig')))
|
||||
cols = list(rows[0].keys())
|
||||
hit = 0
|
||||
for r in rows:
|
||||
k = (key(r['院校']), r['专业代码'])
|
||||
o = off.get(k)
|
||||
if o:
|
||||
hit += 1
|
||||
r['官方目录拟接收推免'] = o[0]
|
||||
r['官方未截断部分'] = o[1]
|
||||
r['官方方向行数'] = o[2]
|
||||
r['官方含截断'] = '是' if o[3] else ''
|
||||
r['公示减官方目录'] = int(r['推免已接收']) - o[0]
|
||||
else:
|
||||
for c in ADD:
|
||||
r[c] = ''
|
||||
r['公示减官方目录'] = ''
|
||||
new_cols = cols[:5] + [c for c in ADD if c not in cols] + cols[5:]
|
||||
with io.open(TBL, 'w', newline='', encoding='utf-8-sig') as f:
|
||||
w = csv.DictWriter(f, fieldnames=new_cols, extrasaction='ignore')
|
||||
w.writeheader()
|
||||
w.writerows(rows)
|
||||
|
||||
L = ['=== 占比表接入目录口径 ===']
|
||||
L.append('目录口径已覆盖 %d 校;占比表 %d 行里有 %d 行拿到官方数(其余是该校方向未取到)' % (
|
||||
nsch, len(rows), hit))
|
||||
d = [int(r['公示减官方目录']) for r in rows if r['公示减官方目录'] != '']
|
||||
eq = sum(1 for x in d if x == 0)
|
||||
L.append('两口径都有的 %d 行:相等 %d 行、公示更高 %d 行、公示更低 %d 行 → '
|
||||
'目录口径不能替代公示口径,只做计划上限' % (len(d), eq, sum(1 for x in d if x > 0),
|
||||
sum(1 for x in d if x < 0)))
|
||||
tr = sum(1 for r in rows if r['官方含截断'] == '是')
|
||||
L.append('官方数受第1页10条截断影响的行=%d(这些行的官方数是下界,不是全量)' % tr)
|
||||
L.append('列顺序=%s' % ','.join(new_cols))
|
||||
open(r'D:/择校/运行暂存/agent_ingest_20261101/official_cols_20261102.txt', 'w',
|
||||
encoding='utf-8').write('\n'.join(L) + '\n')
|
||||
print('\n'.join(L))
|
||||
@@ -0,0 +1,95 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""官方目录推免数 vs 院校公示推免数 交叉核对(只读,分列不合并)。
|
||||
|
||||
两张表口径不同:
|
||||
chsi_2027_tuimian_official = 学校编 2027 目录时填的「拟接收」推免数;
|
||||
y2027_tuimian_2027/_major = 推免接收名单公示出来的实际人数(时点更晚)。
|
||||
差值方向本身就是信息(官方留了余量 / 公示超出计划),但不能相加或取平均,
|
||||
所以这里只输出对照与分档,不动任何一张表的值。
|
||||
"""
|
||||
import collections
|
||||
import csv
|
||||
import io
|
||||
import re
|
||||
import sqlite3
|
||||
|
||||
ZX = r'D:/择校/数据库/zexiao_2027.db'
|
||||
OUT = r'D:/择校/审计报告/0856_推免两口径对照_20261102.csv'
|
||||
LOG = r'D:/择校/运行暂存/agent_ingest_20261101/tuimian_two_calibers.txt'
|
||||
|
||||
|
||||
def key(s):
|
||||
"""校名归一:全角括号转半角、去空白(研招网用半角,公示常写全角)。"""
|
||||
s = re.sub(r'\s+', '', str(s or ''))
|
||||
return s.replace('(', '(').replace(')', ')').replace(':', ':')
|
||||
|
||||
|
||||
z = sqlite3.connect('file:' + ZX + '?mode=ro', uri=True)
|
||||
off = collections.defaultdict(lambda: [0, 0, 0]) # (校,码) -> [全部, 未截断, 方向行数]
|
||||
for dwmc, zydm, t, tr in z.execute(
|
||||
"select dwmc,zydm,tuimian,truncated_page1_only from chsi_2027_tuimian_official "
|
||||
"where zydm like '0856%'"):
|
||||
k = (key(dwmc), zydm)
|
||||
off[k][0] += t or 0
|
||||
if not tr:
|
||||
off[k][1] += t or 0
|
||||
off[k][2] += 1
|
||||
|
||||
pub = collections.defaultdict(int)
|
||||
pub_detail = collections.defaultdict(list)
|
||||
for t, vc in [('y2027_tuimian_2027', 'value'), ('y2027_tuimian_major', 'received_recommend_count')]:
|
||||
for s, m, v in z.execute("select school, major_code, %s from %s" % (vc, t)):
|
||||
if re.match(r'^\d+$', str(v).strip()) and str(m).startswith('0856'):
|
||||
pub[(key(s), str(m).strip())] += int(v)
|
||||
pub_detail[(key(s), str(m).strip())].append(t)
|
||||
z.close()
|
||||
|
||||
L = []
|
||||
L.append('=== 0856 推免两口径对照 ===')
|
||||
L.append('官方目录口径键=%d(覆盖 %d 校) 公示口径键=%d(覆盖 %d 校)' % (
|
||||
len(off), len({k[0] for k in off}), len(pub), len({k[0] for k in pub})))
|
||||
both = sorted(set(off) & set(pub))
|
||||
only_off = sorted(set(off) - set(pub))
|
||||
only_pub = sorted(set(pub) - set(off))
|
||||
L.append('两口径都有=%d 键 只有官方=%d 键 只有公示=%d 键' % (len(both), len(only_off), len(only_pub)))
|
||||
band = collections.Counter()
|
||||
rows = []
|
||||
for k in sorted(set(off) | set(pub)):
|
||||
o = off.get(k, [0, 0, 0])
|
||||
p = pub.get(k, 0)
|
||||
if k in both:
|
||||
d = p - o[0]
|
||||
if d == 0:
|
||||
band['相等'] += 1
|
||||
elif d > 0:
|
||||
band['公示>官方'] += 1
|
||||
else:
|
||||
band['公示<官方'] += 1
|
||||
rows.append({'院校': k[0], '专业代码': k[1],
|
||||
'官方拟接收合计': o[0] if k in off else '',
|
||||
'官方未截断部分': o[1] if k in off else '',
|
||||
'官方方向行数': o[2] if k in off else '',
|
||||
'公示实际接收': p if k in pub else '',
|
||||
'差(公示-官方)': (p - o[0]) if k in both else '',
|
||||
'覆盖情况': ('两口径都有' if k in both else
|
||||
('只有官方目录' if k in off else '只有公示'))})
|
||||
L.append('两口径都有键上的关系: %s' % band.most_common())
|
||||
L.append('')
|
||||
L.append('差值最大的 12 键(公示 − 官方):')
|
||||
for r in sorted([x for x in rows if x['差(公示-官方)'] != ''],
|
||||
key=lambda x: -abs(x['差(公示-官方)']))[:12]:
|
||||
L.append(' %-14s %s 官方=%-5s 公示=%-5s 差=%-6s 官方方向行=%s' % (
|
||||
r['院校'], r['专业代码'], r['官方拟接收合计'], r['公示实际接收'],
|
||||
r['差(公示-官方)'], r['官方方向行数']))
|
||||
L.append('')
|
||||
L.append('只有官方没有公示的校(前 12,是后续公示名单的靶子):')
|
||||
for k in only_off[:12]:
|
||||
L.append(' %-16s %s 官方拟接收=%d' % (k[0][:16], k[1], off[k][0]))
|
||||
with io.open(OUT, 'w', newline='', encoding='utf-8-sig') as f:
|
||||
w = csv.DictWriter(f, fieldnames=list(rows[0].keys()))
|
||||
w.writeheader()
|
||||
w.writerows(rows)
|
||||
L.append('')
|
||||
L.append('对照表写入 %s(%d 行)' % (OUT, len(rows)))
|
||||
open(LOG, 'w', encoding='utf-8').write('\n'.join(L) + '\n')
|
||||
print('\n'.join(L))
|
||||
@@ -0,0 +1,171 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""解析研招网 2027 目录方向响应里的官方推免数,落独立新表。
|
||||
|
||||
为什么单独一张表而不并进 y2027_tuimian_2027:
|
||||
- 这张是**目录口径**(学校编计划时填的拟接收推免数,ssjstmrs,方向级);
|
||||
- y2027_tuimian_2027 是**公示口径**(推免接收名单实际公示的人数);
|
||||
- 两者时间点和含义都不同(清华的备注就写着「招生人数后续可能调整」),
|
||||
合并会让交付层的数无法回溯,所以分列两表,交叉核对只做差值分布。
|
||||
|
||||
自洽性实测:恒有 nzsrs = ssjstmrs + nzsrsstr 里的「不含推免」数
|
||||
(清华深研院 115 = 51 + 64、材料学院 20 = 16 + 4),所以 ssjstmrs 是方向级值,
|
||||
不是学院级重复值 —— 学院级要自己按 yxsdm 汇总。
|
||||
"""
|
||||
import collections
|
||||
import glob
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sqlite3
|
||||
import time
|
||||
|
||||
ZX = r'D:/择校/数据库/zexiao_2027.db'
|
||||
RAW = r'D:/择校/运行暂存/zexiao_cache/chsi_2027/raw/directions'
|
||||
TAB = 'chsi_2027_tuimian_official'
|
||||
DIAG = r'D:/择校/运行暂存/agent_ingest_20261101/official_tuimian_diag.txt'
|
||||
REFUSE = re.compile(r'访问太频繁|请登录|参数错误|频繁|验证|无权|失败')
|
||||
CODE = re.compile(r'^((?:0[1-9]|1[0-4])\d{4}|\d{4}[A-Z]\d{1,2})$')
|
||||
|
||||
|
||||
def norm(s):
|
||||
return re.sub(r'\s+', '', str(s or ''))
|
||||
|
||||
|
||||
def parse_excl(s):
|
||||
"""nzsrsstr 有三种粒度前缀,返回 (口径, 不含推免数, 原文)。"""
|
||||
s = str(s or '')
|
||||
m = re.search(r'(专业|一级学科|院系所)\s*[::]\s*(\d+)', s)
|
||||
if m:
|
||||
return m.group(1), int(m.group(2)), s
|
||||
m = re.search(r'研究方向\s*[::]\s*(\d+)', s)
|
||||
if m:
|
||||
return '研究方向', int(m.group(1)), s
|
||||
return '', None, s
|
||||
|
||||
|
||||
def main():
|
||||
files = sorted(glob.glob(os.path.join(RAW, '*_0856*.json')))
|
||||
rows, bad_refusal, n_trunc, n_item = [], 0, 0, 0
|
||||
seen = set()
|
||||
for p in files:
|
||||
try:
|
||||
d = json.load(open(p, encoding='utf-8'))
|
||||
except Exception:
|
||||
continue
|
||||
m = d.get('msg')
|
||||
if isinstance(m, str):
|
||||
if REFUSE.search(m):
|
||||
bad_refusal += 1
|
||||
continue
|
||||
if not isinstance(m, dict):
|
||||
continue
|
||||
lst = m.get('list') or []
|
||||
tc = m.get('totalCount') or 0
|
||||
if tc > len(lst):
|
||||
n_trunc += 1
|
||||
for it in lst:
|
||||
if not isinstance(it, dict):
|
||||
continue
|
||||
zydm = norm(it.get('zydm'))
|
||||
if not CODE.match(zydm):
|
||||
continue
|
||||
scope, excl, raw = parse_excl(it.get('nzsrsstr'))
|
||||
ssj = it.get('ssjstmrs')
|
||||
nz = it.get('nzsrs')
|
||||
try:
|
||||
ssj = int(str(ssj))
|
||||
except Exception:
|
||||
ssj = None
|
||||
try:
|
||||
nz = int(str(nz))
|
||||
except Exception:
|
||||
nz = None
|
||||
k = (norm(it.get('dwdm')), zydm, norm(it.get('yxsdm')), norm(it.get('yjfxdm')),
|
||||
norm(it.get('xxfs')))
|
||||
if k in seen:
|
||||
continue
|
||||
seen.add(k)
|
||||
n_item += 1
|
||||
rows.append(dict(
|
||||
dwdm=k[0], dwmc=norm(it.get('dwmc')), zydm=zydm, zymc=norm(it.get('zymc')),
|
||||
yxsdm=k[2], yxsmc=norm(it.get('yxsmc')), yjfxdm=k[3],
|
||||
yjfxmc=norm(it.get('yjfxmc')), xxfs=k[4], xxfsmc=norm(it.get('xxfsmc')),
|
||||
tuimian=ssj, nzsrs=nz, excl=excl, scope=scope, raw=raw,
|
||||
truncated=1 if tc > len(lst) else 0, zybz=norm(it.get('zybz'))[:200],
|
||||
source_file=os.path.basename(p),
|
||||
api='zsml/rs/yjfxs.do'))
|
||||
z = sqlite3.connect(ZX, timeout=600)
|
||||
z.execute('PRAGMA busy_timeout=600000')
|
||||
before = z.execute('select count(*) from %s' % TAB).fetchone()[0] if z.execute(
|
||||
"select 1 from sqlite_master where name=?", (TAB,)).fetchone() else None
|
||||
if before is None:
|
||||
z.execute('''CREATE TABLE %s (
|
||||
"dwdm" TEXT,"dwmc" TEXT,"zydm" TEXT,"zymc" TEXT,"yxsdm" TEXT,"yxsmc" TEXT,
|
||||
"yjfxdm" TEXT,"yjfxmc" TEXT,"xxfs" TEXT,"xxfsmc" TEXT,
|
||||
"tuimian" INTEGER,"nzsrs" INTEGER,"excl_num" INTEGER,"quota_scope" TEXT,
|
||||
"nzsrsstr" TEXT,"truncated_page1_only" INTEGER,"zybz" TEXT,
|
||||
"source_file" TEXT,"api" TEXT,"_ingested_at" TEXT,
|
||||
PRIMARY KEY ("dwdm","zydm","yxsdm","yjfxdm","xxfs"))''' % TAB)
|
||||
now = time.strftime('%Y-%m-%dT%H:%M:%S')
|
||||
COLS = ['dwdm', 'dwmc', 'zydm', 'zymc', 'yxsdm', 'yxsmc', 'yjfxdm', 'yjfxmc', 'xxfs',
|
||||
'xxfsmc', 'tuimian', 'nzsrs', 'excl_num', 'quota_scope', 'nzsrsstr',
|
||||
'truncated_page1_only', 'zybz', 'source_file', 'api', '_ingested_at']
|
||||
ins = 'INSERT OR REPLACE INTO %s ("%s") VALUES (%s)' % (
|
||||
TAB, '","'.join(COLS), ','.join('?' * len(COLS)))
|
||||
n = 0
|
||||
for r in rows:
|
||||
z.execute(ins, (r['dwdm'], r['dwmc'], r['zydm'], r['zymc'], r['yxsdm'], r['yxsmc'],
|
||||
r['yjfxdm'], r['yjfxmc'], r['xxfs'], r['xxfsmc'], r['tuimian'],
|
||||
r['nzsrs'], r['excl'], r['scope'], r['raw'], r['truncated'],
|
||||
r['zybz'], r['source_file'], r['api'], now))
|
||||
n += 1
|
||||
tot = z.execute('select count(*) from %s' % TAB).fetchone()[0]
|
||||
sch = z.execute('select count(distinct dwdm) from %s' % TAB).fetchone()[0]
|
||||
has = z.execute('select count(*) from %s where tuimian is not null' % TAB).fetchone()[0]
|
||||
pos = z.execute('select count(*) from %s where coalesce(tuimian,0)>0' % TAB).fetchone()[0]
|
||||
s2 = z.execute("select count(*), sum(tuimian) from %s where tuimian>0 and "
|
||||
"truncated_page1_only=0" % TAB).fetchone()
|
||||
self_ok = z.execute('select count(*) from %s where nzsrs is not null and excl_num is not null '
|
||||
'and nzsrs = tuimian + excl_num' % TAB).fetchone()[0]
|
||||
self_bad = z.execute('select count(*) from %s where nzsrs is not null and excl_num is not null '
|
||||
'and nzsrs <> tuimian + excl_num' % TAB).fetchone()[0]
|
||||
meta = ('研招网 zsml/rs/yjfxs.do 2027 硕士目录',
|
||||
'官方目录口径的拟接收推免数 ssjstmrs;与公示口径 y2027_tuimian_2027 分列,'
|
||||
'truncated_page1_only=1 表示该专业方向数超第1页10条未取全')
|
||||
if z.execute('select 1 from dim_table_meta where table_name=?', (TAB,)).fetchone():
|
||||
z.execute('update dim_table_meta set rows=?, grain=?, year_scope=?, school_col=?, '
|
||||
'major_col=?, source=?, purpose=?, updated_at=? where table_name=?',
|
||||
(tot, 'G5 方向级(学院×研究方向)', '2027', 'dwmc', 'zydm',
|
||||
meta[0], meta[1], now, TAB))
|
||||
else:
|
||||
z.execute('INSERT INTO dim_table_meta VALUES (?,?,?,?,?,?,?,?,?,?,?)',
|
||||
(TAB, tot, 'G5 方向级(学院×研究方向)', '2027', 'dwmc', 'zydm', 0, 0,
|
||||
meta[0], meta[1], now))
|
||||
z.execute('INSERT INTO changelog VALUES (?,?,?,?,?,?)',
|
||||
(time.strftime('%Y-%m-%d %H:%M:%S'), 'chsi_tuimian_official_20261102', TAB,
|
||||
'insert', n,
|
||||
'解析 %d 份 0856 方向响应写入 %d 行;拒绝响应 %d 份、第1页截断 %d 份;'
|
||||
'自洽校验 nzsrs=tuimian+excl 命中 %d / 不符 %d'
|
||||
% (len(files), n, bad_refusal, n_trunc, self_ok, self_bad)))
|
||||
z.commit()
|
||||
L = ['=== 官方目录推免数入库 %s ===' % time.strftime('%Y-%m-%d %H:%M')]
|
||||
L.append('响应文件=%d 解析条目=%d 写入=%d 表内合计=%d 覆盖校=%d' % (
|
||||
len(files), n_item, n, tot, sch))
|
||||
L.append('tuimian 非空=%d 其中 >0 =%d' % (has, pos))
|
||||
L.append('未截断且 tuimian>0:行=%d 人数合计=%s' % (s2[0], s2[1]))
|
||||
L.append('自洽校验 nzsrs = tuimian + excl_num:命中=%d 不符=%d(不符多因方向行给的是专业级名额)' % (
|
||||
self_ok, self_bad))
|
||||
L.append('第1页截断文件=%d 拒绝响应=%d' % (n_trunc, bad_refusal))
|
||||
L.append('')
|
||||
L.append('按校看官方推免数(未截断的方向行汇总,前 14):')
|
||||
q = ("select dwmc, sum(tuimian) t, count(distinct zydm) c from %s "
|
||||
"where truncated_page1_only=0 and coalesce(tuimian,0)>0 group by 1 order by t desc limit 14" % TAB)
|
||||
for r in z.execute(q):
|
||||
L.append(' %-16s 官方拟接收推免=%-6d 涉及码=%d' % (r[0][:16], r[1], r[2]))
|
||||
z.close()
|
||||
open(DIAG, 'w', encoding='utf-8').write('\n'.join(L) + '\n')
|
||||
print('\n'.join(L))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in new issue
Block a user