From f7b95e5e7f4d45cb1f37561ca4bddef6e4c4070e Mon Sep 17 00:00:00 2001 From: Your Name Date: Mon, 2 Nov 2026 02:08:13 +0800 Subject: [PATCH] =?UTF-8?q?=E8=A1=A5=E9=BD=900856=E5=AE=9A=E5=90=91?= =?UTF-8?q?=E6=89=B9=E7=9A=84=E6=8A=93=E5=8F=96=E4=B8=8E=E9=A1=B5=E9=9D=A2?= =?UTF-8?q?=E8=A1=A8=E6=A0=BC=E8=A7=A3=E6=9E=90=E9=93=BE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 三个判据太窄的地方一起放开:附件不止 PDF(还有 xls/docx/zip), 表格可能直接排在页面 HTML 里,个别院校站证书链不全要能降级重试并留痕。 第一版因此把 41 所报成「页面无 PDF」,那是判据问题不是学校没数据。 另修文件名 bug:用 [^0-9A-Za-z] 清洗中文校名会把名字剥成空串, 落盘全成 _0_xxx.pdf,回查时认不出学校;改按 Windows 非法字符过滤并保留中文。 --- 脚本/名单解析/enrich_tm0856_manifest.py | 44 +++++ 脚本/名单解析/fetch_tm0856_v2.py | 238 ++++++++++++++++++++++++ 脚本/名单解析/merge_tm0856.py | 103 ++++++++++ 脚本/名单解析/parse_html_rosters.py | 145 +++++++++++++++ 4 files changed, 530 insertions(+) create mode 100644 脚本/名单解析/enrich_tm0856_manifest.py create mode 100644 脚本/名单解析/fetch_tm0856_v2.py create mode 100644 脚本/名单解析/merge_tm0856.py create mode 100644 脚本/名单解析/parse_html_rosters.py diff --git a/脚本/名单解析/enrich_tm0856_manifest.py b/脚本/名单解析/enrich_tm0856_manifest.py new file mode 100644 index 000000000..ab4df14eb --- /dev/null +++ b/脚本/名单解析/enrich_tm0856_manifest.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- +"""把抓取结果与探测清单合成解析器认得的 manifest。 + +解析器按 local 文件名回查校名/年份/来源,而名单类型只能从标题判断 +(URL 常是 /info/1004/9004.htm 这种无语义路径),所以要把 roster_title 带进来。 +""" +import csv +import os + +BASE = r'D:/择校/运行暂存/agent_ingest_20261101/tm0856' +PROBE = r'D:/择校/运行暂存/agent_search_20261101/tuimian_0856/probe_full.csv' +OUT = os.path.join(BASE, 'fetch_manifest_enriched.csv') + + +def main(): + by_school = {} + for r in csv.DictReader(open(PROBE, encoding='utf-8-sig')): + by_school.setdefault(r['school'], r) + rows = list(csv.DictReader(open(os.path.join(BASE, 'fetch_manifest.csv'), encoding='utf-8-sig'))) + out = [] + for r in rows: + if r['status'] != 'ok' or not (r.get('local') or '').strip(): + continue + p = by_school.get(r['school'], {}) + title = (p.get('roster_title') or '') + ' ' + (p.get('roster_url') or '') + kind = '推免拟录取名单' if any(k in title for k in ('推免', '免试', '免推', 'tms', 'tmnlq')) \ + else '复试名单' + out.append(dict(school=r['school'], year=r['year'], kind=kind, + local=r['local'], url=r['pdf_url'], page_url=r['page_url'], + status='ok', bytes=r['bytes'], pages=r['pages'], rows=r['rows'], + sha256='', content_type='application/pdf', error='')) + cols = list(out[0].keys()) if out else [] + with open(OUT, 'w', newline='', encoding='utf-8-sig') as f: + w = csv.DictWriter(f, fieldnames=cols) + w.writeheader() + w.writerows(out) + import collections + print('可用=%d 校=%d' % (len(out), len({r['school'] for r in out}))) + print('名单类型:', dict(collections.Counter(r['kind'] for r in out))) + print('写入', OUT) + + +if __name__ == '__main__': + main() diff --git a/脚本/名单解析/fetch_tm0856_v2.py b/脚本/名单解析/fetch_tm0856_v2.py new file mode 100644 index 000000000..f0acac761 --- /dev/null +++ b/脚本/名单解析/fetch_tm0856_v2.py @@ -0,0 +1,238 @@ +# -*- coding: utf-8 -*- +"""按 agent 探到的 67 个 2027 推免公示页抓数据:页面 -> 附件 -> 表格缓存。 + +第一版只认 .pdf 附件,结果 41 所报「页面无 PDF」——那是判据太窄,不是学校没数据。 +名单可能在这几种载体里,逐个都要认: + 1. 附件是 xls/xlsx/docx/zip/rar 而非 pdf; + 2. 表格直接排在页面 HTML 里(广外那 95 行逐专业线就是这么来的); + 3. 站点证书链不全,裸请求报 SSLError(单独降级重试并留痕,不因此判该校无数据)。 +名单年份不是 2027 的一律不收:混进 2027 列会直接算错推免占比。 +文件名保留中文校名,只替换 Windows 非法字符——上一版用 [^0-9A-Za-z] 过滤, +中文校名被剥成空串,落盘全成了 _0_xxx.pdf,回查时对不上学校。 +磁盘预算 300MB,单文件上限 8MB,每校最多 3 份附件。 +""" +import csv +import html as htmllib +import json +import os +import re +import threading +from concurrent.futures import ThreadPoolExecutor + +import fitz +import requests +import urllib3 + +SRC = r'D:/择校/运行暂存/agent_search_20261101/tuimian_0856/probe_full.csv' +BASE = r'D:/择校/运行暂存/agent_ingest_20261101/tm0856' +RAW = os.path.join(BASE, 'raw') +CACHE = os.path.join(BASE, 'grid_cache') +MANIFEST = os.path.join(BASE, 'fetch_manifest.csv') +UA = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 ' + 'Chrome/120 Safari/537.36'} +MAX_FILE = 8 * 1024 * 1024 +BUDGET = [300 * 1024 * 1024] +ATTACH_EXT = r'(?:pdf|xls|xlsx|doc|docx|zip|rar|wps)' +lock = threading.Lock() +OLE = bytes([0xD0, 0xCF, 0x11, 0xE0]) +ZIP2 = (bytes([0x50, 0x4B, 0x03, 0x04]), bytes([0x50, 0x4B, 0x05, 0x06])) + + +def decode(b): + for e in ('utf-8', 'gbk', 'gb18030'): + try: + return b.decode(e) + except Exception: + continue + return b.decode('utf-8', 'ignore') + + +def safe(s): + return re.sub(r'[\\/:*?"<>|\s]+', '', s)[:14] or 'x' + + +def attachment_links(page_url, body): + res, seen = [], set() + scheme, rest = page_url.split('://', 1) + host = rest.split('/', 1)[0] + for m in re.finditer(r'href=["\']([^"\']+)["\']', body, re.I): + u = htmllib.unescape(m.group(1).strip()) + if not u or u.lower().startswith(('javascript:', 'mailto:', '#', 'data:')): + continue + if not re.search(r'\.' + ATTACH_EXT + r'($|\?)', u, re.I): + continue + if u.startswith('//'): + u = scheme + ':' + u + elif u.startswith('/'): + u = '%s://%s%s' % (scheme, host, u) + elif not u.startswith('http'): + u = page_url.rsplit('/', 1)[0] + '/' + u + if u not in seen: + seen.add(u) + res.append(u) + return res + + +def rows_from_words(page, ytol=4.0): + """无框线表格的兜底重建:按 y 聚类成行、x 排序成列。""" + ws = page.get_text('words') + if not ws: + return [] + ws.sort(key=lambda w: (w[1], w[0])) + rows, cur, ly = [], [], None + for w in ws: + y = (w[1] + w[3]) / 2.0 + if ly is None or abs(y - ly) <= ytol: + cur.append(w) + ly = y if ly is None else (ly + y) / 2.0 + else: + rows.append(cur) + cur, ly = [w], y + if cur: + rows.append(cur) + return [[w[4] for w in sorted(r, key=lambda w: w[0])] for r in rows] + + +def grid_of_pdf(path): + doc = fitz.open(path) + pages = [] + for page in doc: + try: + tabs = list(page.find_tables().tables) + except Exception: + tabs = [] + g = [] + for tb in tabs: + try: + for row in tb.extract(): + g.append([('' if c is None else str(c)) for c in row]) + except Exception: + continue + rebuilt = rows_from_words(page) + n_re = sum(1 for r in rebuilt if any(str(c).strip() for c in r)) + if n_re > sum(1 for r in g if any(str(c).strip() for c in r)): + g = rebuilt # 靠排版对齐画的表,find_tables 只会给一排空格 + pages.append(g) + n = doc.page_count + doc.close() + return pages, n + + +def get(url, referer=None, verify=True): + h = dict(UA) + if referer: + h['Referer'] = referer + return requests.get(url, headers=h, timeout=60, stream=True, verify=verify) + + +def work(row): + school, page_url = row['school'], row['roster_url'] + year = row.get('roster_year') or '' + rec = dict(school=school, year=year, page_url=page_url, pdf_url='', local='', + status='', bytes=0, pages=0, rows=0, note='') + if year != '2027': + rec['status'] = 'wrong_year_%s' % year + return rec + try: + try: + pr = get(page_url) + except requests.exceptions.SSLError: + pr = get(page_url, verify=False) + rec['note'] = '证书链不全已降级重试' + if pr.status_code != 200: + rec['status'] = 'page_http_%s' % pr.status_code + return rec + body = decode(pr.content) + links = attachment_links(page_url, body) + if not links: + name = safe(school) + '_page.html' + open(os.path.join(RAW, name), 'wb').write(pr.content) + rec.update(local=name, status='page_only', bytes=len(pr.content)) + print('pg %-16s 页面已存(无附件,待 HTML 解析)' % school[:16]) + return rec + for i, u in enumerate(links[:3]): + with lock: + if BUDGET[0] <= 0: + rec['status'] = 'disk_budget' + return rec + try: + rr = get(u, referer=page_url) + except requests.exceptions.SSLError: + rr = get(u, referer=page_url, verify=False) + if rr.status_code != 200: + continue + ext = re.split(r'[?#]', u.rsplit('.', 1)[-1])[0].lower() + if ext not in ('pdf', 'xls', 'xlsx', 'doc', 'docx', 'zip', 'rar', 'wps'): + ext = 'bin' + name = '%s_%d_%s.%s' % (safe(school), i, re.sub(r'\W', '', u)[-8:], ext) + dst = os.path.join(RAW, name) + tot = 0 + with open(dst, 'wb') as f: + for ch in rr.iter_content(65536): + tot += len(ch) + if tot > MAX_FILE: + break + f.write(ch) + head = open(dst, 'rb').read(8) + if not (b'%PDF' in head or head[:4] == OLE or head[:4] in ZIP2 + or head[:3] == b'Rar'): + os.remove(dst) # 端点回的多半是登录页或 HTML 跳转 + continue + with lock: + BUDGET[0] -= tot + if b'%PDF' not in head: + rec.update(pdf_url=u, local=name, status='attachment_nonpdf', bytes=tot, + note=(rec['note'] + ';非PDF附件待解析').strip(';')) + print('att %-16s %s %.1fMB 非PDF,已留档' % (school[:16], ext, tot / 2 ** 20)) + return rec + pages, npg = grid_of_pdf(dst) + nb = sum(1 for g in pages for x in g if any(str(c).strip() for c in x)) + json.dump({'file': name, 'pages': pages}, + open(os.path.join(CACHE, name.rsplit('.', 1)[0] + '.json'), + 'w', encoding='utf-8'), ensure_ascii=False) + rec.update(pdf_url=u, local=name, status='ok' if nb > 5 else 'no_table_rows', + bytes=tot, pages=npg, rows=nb) + print('ok %-16s pages=%-4d rows=%-5d %.1fMB' % ( + school[:16], npg, nb, tot / 2 ** 20)) + return rec + rec['status'] = 'all_attachments_failed' + except Exception as e: + rec['status'] = 'err:%s' % type(e).__name__ + print('-- %-16s %s' % (school[:16], rec['status'])) + return rec + + +def main(): + urllib3.disable_warnings() + os.makedirs(RAW, exist_ok=True) + os.makedirs(CACHE, exist_ok=True) + done = set() + if os.path.exists(MANIFEST): + for r in csv.DictReader(open(MANIFEST, encoding='utf-8-sig')): + if r.get('status') == 'ok': + done.add(r['school']) + rows = [r for r in csv.DictReader(open(SRC, encoding='utf-8-sig')) + if r['verdict'] == 'roster_found' and r['school'] not in done] + print('待取 %d 校(跳过已成功 %d)' % (len(rows), len(done))) + with ThreadPoolExecutor(max_workers=6) as ex: + out = list(ex.map(work, rows)) + old = [] + if os.path.exists(MANIFEST): + old = [r for r in csv.DictReader(open(MANIFEST, encoding='utf-8-sig'))] + allrows = old + out + cols = ['school', 'year', 'page_url', 'pdf_url', 'local', 'status', 'bytes', + 'pages', 'rows', 'note'] + with open(MANIFEST, 'w', newline='', encoding='utf-8-sig') as f: + w = csv.DictWriter(f, fieldnames=cols, extrasaction='ignore', restval='') + w.writeheader() + w.writerows(allrows) + import collections + okn = [x for x in allrows if x['status'] == 'ok'] + print('本批成功=%d 累计成功=%d 校 累计表格行=%d' % ( + sum(1 for x in out if x['status'] == 'ok'), len(okn), + sum(int(x.get('rows') or 0) for x in okn))) + print('形态:', dict(collections.Counter(x['status'] for x in allrows))) + + +if __name__ == '__main__': + main() diff --git a/脚本/名单解析/merge_tm0856.py b/脚本/名单解析/merge_tm0856.py new file mode 100644 index 000000000..25a50f806 --- /dev/null +++ b/脚本/名单解析/merge_tm0856.py @@ -0,0 +1,103 @@ +# -*- coding: utf-8 -*- +"""合并 0856 定向批(PDF + 页面)候选,核验后入库 y2027_tuimian_2027。 + +入库前必须挡掉三类假数据: + 1. 校名其实是列名(那 4 个丢了中文前缀的旧文件会抽出「拟录取学院」当学校); + 2. 名单类型不是推免(这批全部来自推免公示页,若被标成复试名单会污染推免占比); + 3. 单专业接收人数超过该校该专业拟招总数(推免不可能吃掉比计划还多的人)。 +""" +import csv +import json +import os +import re +import sqlite3 +import time + +ZX = r'D:/择校/数据库/zexiao_2027.db' +PDF_CAND = r'D:/择校/运行暂存/agent_ingest_20261101/cand_batch.json' +HTML_CAND = r'D:/择校/运行暂存/agent_ingest_20261101/cand_tm0856_html.json' +MANIFEST = r'D:/择校/运行暂存/agent_ingest_20261101/tm0856/fetch_manifest.csv' +OUT = r'D:/择校/运行暂存/agent_ingest_20261101/cand_tm0856_merged.json' +BAD_SCHOOL = re.compile(r'^(序号|拟录取学院|学院代码|专业代码|姓名|考生编号|拟录取|录取)') + + +def main(): + man = {r['local']: r for r in csv.DictReader(open(MANIFEST, encoding='utf-8-sig')) + if r.get('local')} + valid_schools = {r['school'] for r in man.values()} + man = {r['local']: r for r in csv.DictReader(open(MANIFEST, encoding='utf-8-sig')) + if r.get('local')} + valid_schools = {r['school'] for r in man.values() if r.get('school')} + rows = [] + for path in (PDF_CAND, HTML_CAND): + if not os.path.exists(path): + continue + for x in json.load(open(path, encoding='utf-8')): + # 只认清单里登过的原件,别按文件名前缀猜:前缀是中文校名, + # 用 startswith 过滤会把全部正常行挡掉、只留下校名已丢失的旧文件 + if (x.get('local_file') or '') not in man: + continue + rows.append(x) + print('原始候选=%d' % len(rows)) + bad = {} + out = [] + for r in rows: + sch = (r.get('school') or '').strip() + if not sch or BAD_SCHOOL.match(sch) or sch not in valid_schools: + bad['school_invalid'] = bad.get('school_invalid', 0) + 1 + continue + m = man.get(r.get('local_file') or '', {}) + if m.get('kind') and m['kind'] != r.get('list_kind'): + r['list_kind'] = m['kind'] # 以清单登记的名单类型为准,防误标 + # 解析器对名单型输出 person_count,对计数型输出 value,两个都要认: + # 只读 value 会把全部 PDF 行当成非法值拒掉 + v = str(r.get('value') or r.get('person_count') or '').strip() + r['value'] = v + if not v.isdigit() or not (0 < int(v) <= 600): + bad['value_out_of_range'] = bad.get('value_out_of_range', 0) + 1 + continue + if not (r.get('major_code') or r.get('major_name')): + bad['empty_major'] = bad.get('empty_major', 0) + 1 + continue + page = m.get('page_url') or r.get('source_url') or '' + r['source_url'] = page + r['year'] = r.get('year') or m.get('year') or '2027' + r['plan_type'] = '普通推免' + r['target'] = 'tuimian' + r['note'] = (r.get('note') or '') + ';0856定向批' + out.append(r) + # 与该校该专业的拟招总数比对,超出的标 flag 不丢 + z = sqlite3.connect(ZX, timeout=600) + z.execute('PRAGMA busy_timeout=600000') + quota = {} + for dwmc, zydm, s in z.execute("select dwmc,zydm,enroll_str from chsi_2027_dir " + "where trim(coalesce(enroll_str,''))<>''"): + m = re.search(r'(\d+)', s) + if m: + quota[(dwmc, zydm)] = int(m.group(1)) + over = 0 + for r in out: + q = quota.get((r['school'], r.get('major_code') or '')) + if q and int(r['value']) > q: + r['data_flag'] = 'exceeds_quota' + over += 1 + seen = set() + ded = [] + for r in out: + k = (r['school'], r.get('year'), r.get('college'), r.get('college_code'), + r.get('major_code'), r.get('major_name'), r.get('basis')) + if k in seen: + bad['dup_key'] = bad.get('dup_key', 0) + 1 + continue + seen.add(k) + ded.append(r) + json.dump(ded, open(OUT, 'w', encoding='utf-8'), ensure_ascii=False, indent=1) + print('通过=%d 拒绝=%s 超拟招标flag=%d' % (len(ded), bad, over)) + import collections + print('按校:', collections.Counter(r['school'] for r in ded).most_common()) + print('按类型:', dict(collections.Counter(r['list_kind'] for r in ded))) + z.close() + + +if __name__ == '__main__': + main() diff --git a/脚本/名单解析/parse_html_rosters.py b/脚本/名单解析/parse_html_rosters.py new file mode 100644 index 000000000..fafd8d9e6 --- /dev/null +++ b/脚本/名单解析/parse_html_rosters.py @@ -0,0 +1,145 @@ +# -*- coding: utf-8 -*- +"""解析公示页 HTML 里直接排出的表格(无附件的那些学校)。 + +抓回来的 41 所报「页面无 PDF」,但其中不少是把名单或复试线直接排在页面表格里 +(广东外语外贸大学那 95 行逐专业线就是这个形态)。 +列语义一律按表头关键词定,不按位置认:同是「专业」列,有的写「专业代码」、 +有的写「拟录取专业代码」,有的把代码与名称连写在一格。 +""" +import csv +import glob +import html as htmllib +import json +import os +import re + +RAW = r'D:/择校/运行暂存/agent_ingest_20261101/tm0856/raw' +MANIFEST = r'D:/择校/运行暂存/agent_ingest_20261101/tm0856/fetch_manifest.csv' +OUT = r'D:/择校/运行暂存/agent_ingest_20261101/cand_tm0856_html.json' + +CODE_RE = re.compile(r'(?]*>(.*?)', tr, re.S | re.I): + c = htmllib.unescape(re.sub(r'<[^>]+>', ' ', c)).replace('\xa0', ' ') + out.append(re.sub(r'\s+', ' ', c).strip()) + return out + + +def header_map(hdr): + m = {} + for i, raw in enumerate(hdr): + c = raw.replace(' ', '') + if not c: + continue + if 'code' not in m and '专业代码' in c: + m['code'] = i + elif 'name' not in m and re.search(r'(专业|学科)(名称|全称|及名称)$|^(拟录取|录取)?(专业|学科)$|^学科专业$', c): + m['name'] = i + elif 'college' not in m and (re.search(r'^(拟录取|录取)?(学院|院系|院系所|单位)$', c) + or (re.search(r'(学院|院系|单位|研究院)', c) and re.search(r'(名称|全称)', c))): + m['college'] = i + elif 'college_code' not in m and re.search(r'(学院|院系所?|单位)(代码|码)$', c): + m['college_code'] = i + elif 'count' not in m and re.search(r'^(录取数|接收人数|拟接收|已接收人数)$', c): + m['count'] = i + elif 'total' not in m and re.search(r'总分|复试线$', c) and '要求' not in c: + m['total'] = i + elif 'l100' not in m and re.search(r'单科.*(=|=)?100', c) and '>' not in c: + m['l100'] = i + elif 'lover' not in m and re.search(r'单科.*>', c): + m['lover'] = i + elif 'person' not in m and re.search(r'^(考生)?姓名$', c): + m['person'] = i + return m + + +def main(): + man = {} + if os.path.exists(MANIFEST): + for r in csv.DictReader(open(MANIFEST, encoding='utf-8-sig')): + man[r.get('local') or ''] = r + out = [] + for p in sorted(glob.glob(RAW + '/*_page.html')): + fn = os.path.basename(p) + rec = man.get(fn, {}) + school, year = rec.get('school', ''), rec.get('year', '') + if not school: + continue + t = decode(open(p, 'rb').read()) + trs = re.findall(r']*>(.*?)', t, re.S | re.I) + if len(trs) < 4: + print('%-22s %-12s 无表格行(%d)' % (school[:20], fn, len(trs))) + continue + hm, n_rows, n_line = None, 0, 0 + agg = {} + for tr in trs: + cs = cells(tr) + if not any(cs): + continue + j = '|'.join(cs) + if HDR.search(j) and not CODE_RE.search(re.sub(r'(政治|外语|英语|总分|编号|成绩|学院|专业|姓名|线|要求|学习方式|专项|计划|备注|代码|名称|拟录取|录取|单科|满分|序号|层次|类型|码)', '', j)): + cand = header_map(cs) + if cand.get('code') is not None or cand.get('name') is not None or cand.get('count') is not None: + hm = cand + continue + if hm is None: + continue + if any(w in j for w in ('汇总', '合计', '小计')): + continue + + def at(k): + i = hm.get(k) + return cs[i] if (i is not None and i < len(cs)) else '' + code = '' + cv = at('code') + m = CODE_RE.search((cv or '').replace(' ', '')) + if m: + code = m.group(1) + nm = at('name') + if not code and nm: + m2 = CODE_RE.search(nm.replace(' ', '')) + if m2: + code = m2.group(1) + nm = nm.split(code, 1)[-1] + cnt = at('count').strip() + tot = re.sub(r'[^\d]', '', at('total')) + if re.match(r'^\d{1,5}$', cnt) and (code or nm): + n_line += 1 + out.append(dict(school=school, year=year, college=at('college'), + college_code=at('college_code'), major_code=code, + major_name=re.sub(r'^[))、,::\s]+', '', nm or ''), + kind='', study_mode='', value=cnt, basis='count_direct', + source_url=rec.get('page_url', ''), local_file=fn, + note='页面内表格的官方计数')) + elif (code or nm) and (at('person') or tot): + n_rows += 1 + k = (school, year, at('college'), at('college_code'), code, + re.sub(r'^[))、,::\s]+', '', nm or '')) + agg[k] = agg.get(k, 0) + 1 + for k, v in agg.items(): + out.append(dict(school=k[0], year=k[1], college=k[2], college_code=k[3], + major_code=k[4], major_name=k[5], kind='', study_mode='', + value=str(v), basis='roster_derived', + source_url=rec.get('page_url', ''), local_file=fn, + note='页面内名单按专业分组行数')) + print('%-22s 计数行=%-4d 名单行=%-5d 分组键=%d' % ( + school[:22], n_line, n_rows, len(agg))) + json.dump(out, open(OUT, 'w', encoding='utf-8'), ensure_ascii=False, indent=1) + print('合计候选=%d -> %s' % (len(out), OUT)) + + +if __name__ == '__main__': + main()