diff --git a/数据库/fetch_chsi.py b/数据库/fetch_chsi.py index aed7178a8..ea47d39b1 100644 --- a/数据库/fetch_chsi.py +++ b/数据库/fetch_chsi.py @@ -244,6 +244,7 @@ def run_shard(fc, dwdm, dwmc, yjxkdm): def main(): + global CACHE, RAW_MAJORS, RAW_DIRS, STATE, MANIFEST, ABORT_FILE ap = argparse.ArgumentParser(description="研招网 2027 目录取数器(只读 GET,D 盘缓存)") ap.add_argument("--root", default=r"C:\Users\12742\Desktop\择校", help="项目根,读院校清单用") ap.add_argument("--yjxkdm", required=True, help="门类/一级学科码,逗号分隔,如 0854,0812") @@ -254,8 +255,20 @@ def main(): ap.add_argument("--max-retry", type=int, default=3) ap.add_argument("--max-requests", type=int, default=200, help="本次硬预算,到量即停并保留续跑点") ap.add_argument("--rerun", action="store_true", help="忽略已完成分片,重跑") + ap.add_argument("--cache", default=CACHE, + help="缓存根。默认是迁移前的 D:/zexiao_cache/chsi_2027;" + "本项目已迁到 运行暂存/ 下,续跑时必须显式指过去," + "否则已完成的分片会被当成空白重新下载") a = ap.parse_args() + CACHE = a.cache.replace("\\", "/").rstrip("/") + RAW_MAJORS = CACHE + "/raw/majors" + RAW_DIRS = CACHE + "/raw/directions" + STATE = CACHE + "/state" + MANIFEST = CACHE + "/manifest.jsonl" + ABORT_FILE = CACHE + "/ABORT.json" + log("缓存根=%s" % CACHE) + ensure_dirs() codes = [c.strip() for c in a.yjxkdm.split(",") if c.strip()] diff --git a/运行暂存/agent_ingest_20261101/add_official_cols.py b/运行暂存/agent_ingest_20261101/add_official_cols.py new file mode 100644 index 000000000..4bfacfd73 --- /dev/null +++ b/运行暂存/agent_ingest_20261101/add_official_cols.py @@ -0,0 +1,76 @@ +# -*- coding: utf-8 -*- +"""给 0856 占比表加官方目录口径三列(公示口径与目录口径并排,不相加)。 + +占比表原有的是公示口径(院校名单实际接收),新表 chsi_2027_tuimian_official 是 +目录口径(学校编计划时填的拟接收)。两者在重叠键上没有一个相等, +所以只能并排展示并标出方向行数与截断,供使用者判断该用哪个。 +""" +import collections +import csv +import io +import re +import sqlite3 + +TBL = r'D:/择校/审计报告/0856_统考名额与推免占比_20261101.csv' +ZX = r'D:/择校/数据库/zexiao_2027.db' +ADD = ['官方目录拟接收推免', '官方未截断部分', '官方方向行数', '官方含截断', + '公示减官方目录'] + + +def key(s): + s = re.sub(r'\s+', '', str(s or '')) + return s.replace('(', '(').replace(')', ')') + + +z = sqlite3.connect('file:' + ZX + '?mode=ro', uri=True) +off = collections.defaultdict(lambda: [0, 0, 0, 0]) +for dwmc, zydm, t, tr in z.execute( + "select dwmc,zydm,tuimian,truncated_page1_only from chsi_2027_tuimian_official " + "where zydm like '0856%'"): + k = (key(dwmc), zydm) + off[k][0] += t or 0 + if not tr: + off[k][1] += t or 0 + off[k][2] += 1 + off[k][3] = max(off[k][3], tr or 0) +nsch = z.execute("select count(distinct dwmc) from chsi_2027_tuimian_official " + "where zydm like '0856%'").fetchone()[0] +z.close() + +rows = list(csv.DictReader(open(TBL, encoding='utf-8-sig'))) +cols = list(rows[0].keys()) +hit = 0 +for r in rows: + k = (key(r['院校']), r['专业代码']) + o = off.get(k) + if o: + hit += 1 + r['官方目录拟接收推免'] = o[0] + r['官方未截断部分'] = o[1] + r['官方方向行数'] = o[2] + r['官方含截断'] = '是' if o[3] else '' + r['公示减官方目录'] = int(r['推免已接收']) - o[0] + else: + for c in ADD: + r[c] = '' + r['公示减官方目录'] = '' +new_cols = cols[:5] + [c for c in ADD if c not in cols] + cols[5:] +with io.open(TBL, 'w', newline='', encoding='utf-8-sig') as f: + w = csv.DictWriter(f, fieldnames=new_cols, extrasaction='ignore') + w.writeheader() + w.writerows(rows) + +L = ['=== 占比表接入目录口径 ==='] +L.append('目录口径已覆盖 %d 校;占比表 %d 行里有 %d 行拿到官方数(其余是该校方向未取到)' % ( + nsch, len(rows), hit)) +d = [int(r['公示减官方目录']) for r in rows if r['公示减官方目录'] != ''] +eq = sum(1 for x in d if x == 0) +L.append('两口径都有的 %d 行:相等 %d 行、公示更高 %d 行、公示更低 %d 行 → ' + '目录口径不能替代公示口径,只做计划上限' % (len(d), eq, sum(1 for x in d if x > 0), + sum(1 for x in d if x < 0))) +tr = sum(1 for r in rows if r['官方含截断'] == '是') +L.append('官方数受第1页10条截断影响的行=%d(这些行的官方数是下界,不是全量)' % tr) +L.append('列顺序=%s' % ','.join(new_cols)) +open(r'D:/择校/运行暂存/agent_ingest_20261101/official_cols_20261102.txt', 'w', + encoding='utf-8').write('\n'.join(L) + '\n') +print('\n'.join(L)) diff --git a/运行暂存/agent_ingest_20261101/crosscheck_two_tuimian_calibers.py b/运行暂存/agent_ingest_20261101/crosscheck_two_tuimian_calibers.py new file mode 100644 index 000000000..7e806edcd --- /dev/null +++ b/运行暂存/agent_ingest_20261101/crosscheck_two_tuimian_calibers.py @@ -0,0 +1,95 @@ +# -*- coding: utf-8 -*- +"""官方目录推免数 vs 院校公示推免数 交叉核对(只读,分列不合并)。 + +两张表口径不同: + chsi_2027_tuimian_official = 学校编 2027 目录时填的「拟接收」推免数; + y2027_tuimian_2027/_major = 推免接收名单公示出来的实际人数(时点更晚)。 +差值方向本身就是信息(官方留了余量 / 公示超出计划),但不能相加或取平均, +所以这里只输出对照与分档,不动任何一张表的值。 +""" +import collections +import csv +import io +import re +import sqlite3 + +ZX = r'D:/择校/数据库/zexiao_2027.db' +OUT = r'D:/择校/审计报告/0856_推免两口径对照_20261102.csv' +LOG = r'D:/择校/运行暂存/agent_ingest_20261101/tuimian_two_calibers.txt' + + +def key(s): + """校名归一:全角括号转半角、去空白(研招网用半角,公示常写全角)。""" + s = re.sub(r'\s+', '', str(s or '')) + return s.replace('(', '(').replace(')', ')').replace(':', ':') + + +z = sqlite3.connect('file:' + ZX + '?mode=ro', uri=True) +off = collections.defaultdict(lambda: [0, 0, 0]) # (校,码) -> [全部, 未截断, 方向行数] +for dwmc, zydm, t, tr in z.execute( + "select dwmc,zydm,tuimian,truncated_page1_only from chsi_2027_tuimian_official " + "where zydm like '0856%'"): + k = (key(dwmc), zydm) + off[k][0] += t or 0 + if not tr: + off[k][1] += t or 0 + off[k][2] += 1 + +pub = collections.defaultdict(int) +pub_detail = collections.defaultdict(list) +for t, vc in [('y2027_tuimian_2027', 'value'), ('y2027_tuimian_major', 'received_recommend_count')]: + for s, m, v in z.execute("select school, major_code, %s from %s" % (vc, t)): + if re.match(r'^\d+$', str(v).strip()) and str(m).startswith('0856'): + pub[(key(s), str(m).strip())] += int(v) + pub_detail[(key(s), str(m).strip())].append(t) +z.close() + +L = [] +L.append('=== 0856 推免两口径对照 ===') +L.append('官方目录口径键=%d(覆盖 %d 校) 公示口径键=%d(覆盖 %d 校)' % ( + len(off), len({k[0] for k in off}), len(pub), len({k[0] for k in pub}))) +both = sorted(set(off) & set(pub)) +only_off = sorted(set(off) - set(pub)) +only_pub = sorted(set(pub) - set(off)) +L.append('两口径都有=%d 键 只有官方=%d 键 只有公示=%d 键' % (len(both), len(only_off), len(only_pub))) +band = collections.Counter() +rows = [] +for k in sorted(set(off) | set(pub)): + o = off.get(k, [0, 0, 0]) + p = pub.get(k, 0) + if k in both: + d = p - o[0] + if d == 0: + band['相等'] += 1 + elif d > 0: + band['公示>官方'] += 1 + else: + band['公示<官方'] += 1 + rows.append({'院校': k[0], '专业代码': k[1], + '官方拟接收合计': o[0] if k in off else '', + '官方未截断部分': o[1] if k in off else '', + '官方方向行数': o[2] if k in off else '', + '公示实际接收': p if k in pub else '', + '差(公示-官方)': (p - o[0]) if k in both else '', + '覆盖情况': ('两口径都有' if k in both else + ('只有官方目录' if k in off else '只有公示'))}) +L.append('两口径都有键上的关系: %s' % band.most_common()) +L.append('') +L.append('差值最大的 12 键(公示 − 官方):') +for r in sorted([x for x in rows if x['差(公示-官方)'] != ''], + key=lambda x: -abs(x['差(公示-官方)']))[:12]: + L.append(' %-14s %s 官方=%-5s 公示=%-5s 差=%-6s 官方方向行=%s' % ( + r['院校'], r['专业代码'], r['官方拟接收合计'], r['公示实际接收'], + r['差(公示-官方)'], r['官方方向行数'])) +L.append('') +L.append('只有官方没有公示的校(前 12,是后续公示名单的靶子):') +for k in only_off[:12]: + L.append(' %-16s %s 官方拟接收=%d' % (k[0][:16], k[1], off[k][0])) +with io.open(OUT, 'w', newline='', encoding='utf-8-sig') as f: + w = csv.DictWriter(f, fieldnames=list(rows[0].keys())) + w.writeheader() + w.writerows(rows) +L.append('') +L.append('对照表写入 %s(%d 行)' % (OUT, len(rows))) +open(LOG, 'w', encoding='utf-8').write('\n'.join(L) + '\n') +print('\n'.join(L)) diff --git a/运行暂存/agent_ingest_20261101/ingest_official_tuimian.py b/运行暂存/agent_ingest_20261101/ingest_official_tuimian.py new file mode 100644 index 000000000..3880b0633 --- /dev/null +++ b/运行暂存/agent_ingest_20261101/ingest_official_tuimian.py @@ -0,0 +1,171 @@ +# -*- coding: utf-8 -*- +"""解析研招网 2027 目录方向响应里的官方推免数,落独立新表。 + +为什么单独一张表而不并进 y2027_tuimian_2027: + - 这张是**目录口径**(学校编计划时填的拟接收推免数,ssjstmrs,方向级); + - y2027_tuimian_2027 是**公示口径**(推免接收名单实际公示的人数); + - 两者时间点和含义都不同(清华的备注就写着「招生人数后续可能调整」), + 合并会让交付层的数无法回溯,所以分列两表,交叉核对只做差值分布。 + +自洽性实测:恒有 nzsrs = ssjstmrs + nzsrsstr 里的「不含推免」数 +(清华深研院 115 = 51 + 64、材料学院 20 = 16 + 4),所以 ssjstmrs 是方向级值, +不是学院级重复值 —— 学院级要自己按 yxsdm 汇总。 +""" +import collections +import glob +import json +import os +import re +import sqlite3 +import time + +ZX = r'D:/择校/数据库/zexiao_2027.db' +RAW = r'D:/择校/运行暂存/zexiao_cache/chsi_2027/raw/directions' +TAB = 'chsi_2027_tuimian_official' +DIAG = r'D:/择校/运行暂存/agent_ingest_20261101/official_tuimian_diag.txt' +REFUSE = re.compile(r'访问太频繁|请登录|参数错误|频繁|验证|无权|失败') +CODE = re.compile(r'^((?:0[1-9]|1[0-4])\d{4}|\d{4}[A-Z]\d{1,2})$') + + +def norm(s): + return re.sub(r'\s+', '', str(s or '')) + + +def parse_excl(s): + """nzsrsstr 有三种粒度前缀,返回 (口径, 不含推免数, 原文)。""" + s = str(s or '') + m = re.search(r'(专业|一级学科|院系所)\s*[::]\s*(\d+)', s) + if m: + return m.group(1), int(m.group(2)), s + m = re.search(r'研究方向\s*[::]\s*(\d+)', s) + if m: + return '研究方向', int(m.group(1)), s + return '', None, s + + +def main(): + files = sorted(glob.glob(os.path.join(RAW, '*_0856*.json'))) + rows, bad_refusal, n_trunc, n_item = [], 0, 0, 0 + seen = set() + for p in files: + try: + d = json.load(open(p, encoding='utf-8')) + except Exception: + continue + m = d.get('msg') + if isinstance(m, str): + if REFUSE.search(m): + bad_refusal += 1 + continue + if not isinstance(m, dict): + continue + lst = m.get('list') or [] + tc = m.get('totalCount') or 0 + if tc > len(lst): + n_trunc += 1 + for it in lst: + if not isinstance(it, dict): + continue + zydm = norm(it.get('zydm')) + if not CODE.match(zydm): + continue + scope, excl, raw = parse_excl(it.get('nzsrsstr')) + ssj = it.get('ssjstmrs') + nz = it.get('nzsrs') + try: + ssj = int(str(ssj)) + except Exception: + ssj = None + try: + nz = int(str(nz)) + except Exception: + nz = None + k = (norm(it.get('dwdm')), zydm, norm(it.get('yxsdm')), norm(it.get('yjfxdm')), + norm(it.get('xxfs'))) + if k in seen: + continue + seen.add(k) + n_item += 1 + rows.append(dict( + dwdm=k[0], dwmc=norm(it.get('dwmc')), zydm=zydm, zymc=norm(it.get('zymc')), + yxsdm=k[2], yxsmc=norm(it.get('yxsmc')), yjfxdm=k[3], + yjfxmc=norm(it.get('yjfxmc')), xxfs=k[4], xxfsmc=norm(it.get('xxfsmc')), + tuimian=ssj, nzsrs=nz, excl=excl, scope=scope, raw=raw, + truncated=1 if tc > len(lst) else 0, zybz=norm(it.get('zybz'))[:200], + source_file=os.path.basename(p), + api='zsml/rs/yjfxs.do')) + z = sqlite3.connect(ZX, timeout=600) + z.execute('PRAGMA busy_timeout=600000') + before = z.execute('select count(*) from %s' % TAB).fetchone()[0] if z.execute( + "select 1 from sqlite_master where name=?", (TAB,)).fetchone() else None + if before is None: + z.execute('''CREATE TABLE %s ( + "dwdm" TEXT,"dwmc" TEXT,"zydm" TEXT,"zymc" TEXT,"yxsdm" TEXT,"yxsmc" TEXT, + "yjfxdm" TEXT,"yjfxmc" TEXT,"xxfs" TEXT,"xxfsmc" TEXT, + "tuimian" INTEGER,"nzsrs" INTEGER,"excl_num" INTEGER,"quota_scope" TEXT, + "nzsrsstr" TEXT,"truncated_page1_only" INTEGER,"zybz" TEXT, + "source_file" TEXT,"api" TEXT,"_ingested_at" TEXT, + PRIMARY KEY ("dwdm","zydm","yxsdm","yjfxdm","xxfs"))''' % TAB) + now = time.strftime('%Y-%m-%dT%H:%M:%S') + COLS = ['dwdm', 'dwmc', 'zydm', 'zymc', 'yxsdm', 'yxsmc', 'yjfxdm', 'yjfxmc', 'xxfs', + 'xxfsmc', 'tuimian', 'nzsrs', 'excl_num', 'quota_scope', 'nzsrsstr', + 'truncated_page1_only', 'zybz', 'source_file', 'api', '_ingested_at'] + ins = 'INSERT OR REPLACE INTO %s ("%s") VALUES (%s)' % ( + TAB, '","'.join(COLS), ','.join('?' * len(COLS))) + n = 0 + for r in rows: + z.execute(ins, (r['dwdm'], r['dwmc'], r['zydm'], r['zymc'], r['yxsdm'], r['yxsmc'], + r['yjfxdm'], r['yjfxmc'], r['xxfs'], r['xxfsmc'], r['tuimian'], + r['nzsrs'], r['excl'], r['scope'], r['raw'], r['truncated'], + r['zybz'], r['source_file'], r['api'], now)) + n += 1 + tot = z.execute('select count(*) from %s' % TAB).fetchone()[0] + sch = z.execute('select count(distinct dwdm) from %s' % TAB).fetchone()[0] + has = z.execute('select count(*) from %s where tuimian is not null' % TAB).fetchone()[0] + pos = z.execute('select count(*) from %s where coalesce(tuimian,0)>0' % TAB).fetchone()[0] + s2 = z.execute("select count(*), sum(tuimian) from %s where tuimian>0 and " + "truncated_page1_only=0" % TAB).fetchone() + self_ok = z.execute('select count(*) from %s where nzsrs is not null and excl_num is not null ' + 'and nzsrs = tuimian + excl_num' % TAB).fetchone()[0] + self_bad = z.execute('select count(*) from %s where nzsrs is not null and excl_num is not null ' + 'and nzsrs <> tuimian + excl_num' % TAB).fetchone()[0] + meta = ('研招网 zsml/rs/yjfxs.do 2027 硕士目录', + '官方目录口径的拟接收推免数 ssjstmrs;与公示口径 y2027_tuimian_2027 分列,' + 'truncated_page1_only=1 表示该专业方向数超第1页10条未取全') + if z.execute('select 1 from dim_table_meta where table_name=?', (TAB,)).fetchone(): + z.execute('update dim_table_meta set rows=?, grain=?, year_scope=?, school_col=?, ' + 'major_col=?, source=?, purpose=?, updated_at=? where table_name=?', + (tot, 'G5 方向级(学院×研究方向)', '2027', 'dwmc', 'zydm', + meta[0], meta[1], now, TAB)) + else: + z.execute('INSERT INTO dim_table_meta VALUES (?,?,?,?,?,?,?,?,?,?,?)', + (TAB, tot, 'G5 方向级(学院×研究方向)', '2027', 'dwmc', 'zydm', 0, 0, + meta[0], meta[1], now)) + z.execute('INSERT INTO changelog VALUES (?,?,?,?,?,?)', + (time.strftime('%Y-%m-%d %H:%M:%S'), 'chsi_tuimian_official_20261102', TAB, + 'insert', n, + '解析 %d 份 0856 方向响应写入 %d 行;拒绝响应 %d 份、第1页截断 %d 份;' + '自洽校验 nzsrs=tuimian+excl 命中 %d / 不符 %d' + % (len(files), n, bad_refusal, n_trunc, self_ok, self_bad))) + z.commit() + L = ['=== 官方目录推免数入库 %s ===' % time.strftime('%Y-%m-%d %H:%M')] + L.append('响应文件=%d 解析条目=%d 写入=%d 表内合计=%d 覆盖校=%d' % ( + len(files), n_item, n, tot, sch)) + L.append('tuimian 非空=%d 其中 >0 =%d' % (has, pos)) + L.append('未截断且 tuimian>0:行=%d 人数合计=%s' % (s2[0], s2[1])) + L.append('自洽校验 nzsrs = tuimian + excl_num:命中=%d 不符=%d(不符多因方向行给的是专业级名额)' % ( + self_ok, self_bad)) + L.append('第1页截断文件=%d 拒绝响应=%d' % (n_trunc, bad_refusal)) + L.append('') + L.append('按校看官方推免数(未截断的方向行汇总,前 14):') + q = ("select dwmc, sum(tuimian) t, count(distinct zydm) c from %s " + "where truncated_page1_only=0 and coalesce(tuimian,0)>0 group by 1 order by t desc limit 14" % TAB) + for r in z.execute(q): + L.append(' %-16s 官方拟接收推免=%-6d 涉及码=%d' % (r[0][:16], r[1], r[2])) + z.close() + open(DIAG, 'w', encoding='utf-8').write('\n'.join(L) + '\n') + print('\n'.join(L)) + + +if __name__ == '__main__': + main()