补齐0856定向批的抓取与页面表格解析链

三个判据太窄的地方一起放开:附件不止 PDF(还有 xls/docx/zip),
表格可能直接排在页面 HTML 里,个别院校站证书链不全要能降级重试并留痕。
第一版因此把 41 所报成「页面无 PDF」,那是判据问题不是学校没数据。

另修文件名 bug:用 [^0-9A-Za-z] 清洗中文校名会把名字剥成空串,
落盘全成 _0_xxx.pdf,回查时认不出学校;改按 Windows 非法字符过滤并保留中文。
This commit is contained in:
Your Name committed 2026-11-02 02:08:13 +08:00
1 parent b0c289b797
commit f7b95e5e7f
4 files changed
+530

No files matched your search

@@ -0,0 +1,44 @@
# -*- coding: utf-8 -*-
"""把抓取结果与探测清单合成解析器认得的 manifest。
解析器按 local 文件名回查校名/年份/来源,而名单类型只能从标题判断
(URL 常是 /info/1004/9004.htm 这种无语义路径),所以要把 roster_title 带进来。
"""
import csv
import os
BASE = r'D:/择校/运行暂存/agent_ingest_20261101/tm0856'
PROBE = r'D:/择校/运行暂存/agent_search_20261101/tuimian_0856/probe_full.csv'
OUT = os.path.join(BASE, 'fetch_manifest_enriched.csv')
def main():
by_school = {}
for r in csv.DictReader(open(PROBE, encoding='utf-8-sig')):
by_school.setdefault(r['school'], r)
rows = list(csv.DictReader(open(os.path.join(BASE, 'fetch_manifest.csv'), encoding='utf-8-sig')))
out = []
for r in rows:
if r['status'] != 'ok' or not (r.get('local') or '').strip():
continue
p = by_school.get(r['school'], {})
title = (p.get('roster_title') or '') + ' ' + (p.get('roster_url') or '')
kind = '推免拟录取名单' if any(k in title for k in ('推免', '免试', '免推', 'tms', 'tmnlq')) \
else '复试名单'
out.append(dict(school=r['school'], year=r['year'], kind=kind,
local=r['local'], url=r['pdf_url'], page_url=r['page_url'],
status='ok', bytes=r['bytes'], pages=r['pages'], rows=r['rows'],
sha256='', content_type='application/pdf', error=''))
cols = list(out[0].keys()) if out else []
with open(OUT, 'w', newline='', encoding='utf-8-sig') as f:
w = csv.DictWriter(f, fieldnames=cols)
w.writeheader()
w.writerows(out)
import collections
print('可用=%d 校=%d' % (len(out), len({r['school'] for r in out})))
print('名单类型:', dict(collections.Counter(r['kind'] for r in out)))
print('写入', OUT)
if __name__ == '__main__':
main()
+238
View File
@@ -0,0 +1,238 @@
# -*- coding: utf-8 -*-
"""按 agent 探到的 67 个 2027 推免公示页抓数据:页面 -> 附件 -> 表格缓存。
第一版只认 .pdf 附件,结果 41 所报「页面无 PDF」——那是判据太窄,不是学校没数据。
名单可能在这几种载体里,逐个都要认:
1. 附件是 xls/xlsx/docx/zip/rar 而非 pdf;
2. 表格直接排在页面 HTML 里(广外那 95 行逐专业线就是这么来的);
3. 站点证书链不全,裸请求报 SSLError(单独降级重试并留痕,不因此判该校无数据)。
名单年份不是 2027 的一律不收:混进 2027 列会直接算错推免占比。
文件名保留中文校名,只替换 Windows 非法字符——上一版用 [^0-9A-Za-z] 过滤,
中文校名被剥成空串,落盘全成了 _0_xxx.pdf,回查时对不上学校。
磁盘预算 300MB,单文件上限 8MB,每校最多 3 份附件。
"""
import csv
import html as htmllib
import json
import os
import re
import threading
from concurrent.futures import ThreadPoolExecutor
import fitz
import requests
import urllib3
SRC = r'D:/择校/运行暂存/agent_search_20261101/tuimian_0856/probe_full.csv'
BASE = r'D:/择校/运行暂存/agent_ingest_20261101/tm0856'
RAW = os.path.join(BASE, 'raw')
CACHE = os.path.join(BASE, 'grid_cache')
MANIFEST = os.path.join(BASE, 'fetch_manifest.csv')
UA = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 '
'Chrome/120 Safari/537.36'}
MAX_FILE = 8 * 1024 * 1024
BUDGET = [300 * 1024 * 1024]
ATTACH_EXT = r'(?:pdf|xls|xlsx|doc|docx|zip|rar|wps)'
lock = threading.Lock()
OLE = bytes([0xD0, 0xCF, 0x11, 0xE0])
ZIP2 = (bytes([0x50, 0x4B, 0x03, 0x04]), bytes([0x50, 0x4B, 0x05, 0x06]))
def decode(b):
for e in ('utf-8', 'gbk', 'gb18030'):
try:
return b.decode(e)
except Exception:
continue
return b.decode('utf-8', 'ignore')
def safe(s):
return re.sub(r'[\\/:*?"<>|\s]+', '', s)[:14] or 'x'
def attachment_links(page_url, body):
res, seen = [], set()
scheme, rest = page_url.split('://', 1)
host = rest.split('/', 1)[0]
for m in re.finditer(r'href=["\']([^"\']+)["\']', body, re.I):
u = htmllib.unescape(m.group(1).strip())
if not u or u.lower().startswith(('javascript:', 'mailto:', '#', 'data:')):
continue
if not re.search(r'\.' + ATTACH_EXT + r'($|\?)', u, re.I):
continue
if u.startswith('//'):
u = scheme + ':' + u
elif u.startswith('/'):
u = '%s://%s%s' % (scheme, host, u)
elif not u.startswith('http'):
u = page_url.rsplit('/', 1)[0] + '/' + u
if u not in seen:
seen.add(u)
res.append(u)
return res
def rows_from_words(page, ytol=4.0):
"""无框线表格的兜底重建:按 y 聚类成行、x 排序成列。"""
ws = page.get_text('words')
if not ws:
return []
ws.sort(key=lambda w: (w[1], w[0]))
rows, cur, ly = [], [], None
for w in ws:
y = (w[1] + w[3]) / 2.0
if ly is None or abs(y - ly) <= ytol:
cur.append(w)
ly = y if ly is None else (ly + y) / 2.0
else:
rows.append(cur)
cur, ly = [w], y
if cur:
rows.append(cur)
return [[w[4] for w in sorted(r, key=lambda w: w[0])] for r in rows]
def grid_of_pdf(path):
doc = fitz.open(path)
pages = []
for page in doc:
try:
tabs = list(page.find_tables().tables)
except Exception:
tabs = []
g = []
for tb in tabs:
try:
for row in tb.extract():
g.append([('' if c is None else str(c)) for c in row])
except Exception:
continue
rebuilt = rows_from_words(page)
n_re = sum(1 for r in rebuilt if any(str(c).strip() for c in r))
if n_re > sum(1 for r in g if any(str(c).strip() for c in r)):
g = rebuilt # 靠排版对齐画的表,find_tables 只会给一排空格
pages.append(g)
n = doc.page_count
doc.close()
return pages, n
def get(url, referer=None, verify=True):
h = dict(UA)
if referer:
h['Referer'] = referer
return requests.get(url, headers=h, timeout=60, stream=True, verify=verify)
def work(row):
school, page_url = row['school'], row['roster_url']
year = row.get('roster_year') or ''
rec = dict(school=school, year=year, page_url=page_url, pdf_url='', local='',
status='', bytes=0, pages=0, rows=0, note='')
if year != '2027':
rec['status'] = 'wrong_year_%s' % year
return rec
try:
try:
pr = get(page_url)
except requests.exceptions.SSLError:
pr = get(page_url, verify=False)
rec['note'] = '证书链不全已降级重试'
if pr.status_code != 200:
rec['status'] = 'page_http_%s' % pr.status_code
return rec
body = decode(pr.content)
links = attachment_links(page_url, body)
if not links:
name = safe(school) + '_page.html'
open(os.path.join(RAW, name), 'wb').write(pr.content)
rec.update(local=name, status='page_only', bytes=len(pr.content))
print('pg %-16s 页面已存(无附件,待 HTML 解析)' % school[:16])
return rec
for i, u in enumerate(links[:3]):
with lock:
if BUDGET[0] <= 0:
rec['status'] = 'disk_budget'
return rec
try:
rr = get(u, referer=page_url)
except requests.exceptions.SSLError:
rr = get(u, referer=page_url, verify=False)
if rr.status_code != 200:
continue
ext = re.split(r'[?#]', u.rsplit('.', 1)[-1])[0].lower()
if ext not in ('pdf', 'xls', 'xlsx', 'doc', 'docx', 'zip', 'rar', 'wps'):
ext = 'bin'
name = '%s_%d_%s.%s' % (safe(school), i, re.sub(r'\W', '', u)[-8:], ext)
dst = os.path.join(RAW, name)
tot = 0
with open(dst, 'wb') as f:
for ch in rr.iter_content(65536):
tot += len(ch)
if tot > MAX_FILE:
break
f.write(ch)
head = open(dst, 'rb').read(8)
if not (b'%PDF' in head or head[:4] == OLE or head[:4] in ZIP2
or head[:3] == b'Rar'):
os.remove(dst) # 端点回的多半是登录页或 HTML 跳转
continue
with lock:
BUDGET[0] -= tot
if b'%PDF' not in head:
rec.update(pdf_url=u, local=name, status='attachment_nonpdf', bytes=tot,
note=(rec['note'] + ';非PDF附件待解析').strip(';'))
print('att %-16s %s %.1fMB 非PDF,已留档' % (school[:16], ext, tot / 2 ** 20))
return rec
pages, npg = grid_of_pdf(dst)
nb = sum(1 for g in pages for x in g if any(str(c).strip() for c in x))
json.dump({'file': name, 'pages': pages},
open(os.path.join(CACHE, name.rsplit('.', 1)[0] + '.json'),
'w', encoding='utf-8'), ensure_ascii=False)
rec.update(pdf_url=u, local=name, status='ok' if nb > 5 else 'no_table_rows',
bytes=tot, pages=npg, rows=nb)
print('ok %-16s pages=%-4d rows=%-5d %.1fMB' % (
school[:16], npg, nb, tot / 2 ** 20))
return rec
rec['status'] = 'all_attachments_failed'
except Exception as e:
rec['status'] = 'err:%s' % type(e).__name__
print('-- %-16s %s' % (school[:16], rec['status']))
return rec
def main():
urllib3.disable_warnings()
os.makedirs(RAW, exist_ok=True)
os.makedirs(CACHE, exist_ok=True)
done = set()
if os.path.exists(MANIFEST):
for r in csv.DictReader(open(MANIFEST, encoding='utf-8-sig')):
if r.get('status') == 'ok':
done.add(r['school'])
rows = [r for r in csv.DictReader(open(SRC, encoding='utf-8-sig'))
if r['verdict'] == 'roster_found' and r['school'] not in done]
print('待取 %d 校(跳过已成功 %d)' % (len(rows), len(done)))
with ThreadPoolExecutor(max_workers=6) as ex:
out = list(ex.map(work, rows))
old = []
if os.path.exists(MANIFEST):
old = [r for r in csv.DictReader(open(MANIFEST, encoding='utf-8-sig'))]
allrows = old + out
cols = ['school', 'year', 'page_url', 'pdf_url', 'local', 'status', 'bytes',
'pages', 'rows', 'note']
with open(MANIFEST, 'w', newline='', encoding='utf-8-sig') as f:
w = csv.DictWriter(f, fieldnames=cols, extrasaction='ignore', restval='')
w.writeheader()
w.writerows(allrows)
import collections
okn = [x for x in allrows if x['status'] == 'ok']
print('本批成功=%d 累计成功=%d 校 累计表格行=%d' % (
sum(1 for x in out if x['status'] == 'ok'), len(okn),
sum(int(x.get('rows') or 0) for x in okn)))
print('形态:', dict(collections.Counter(x['status'] for x in allrows)))
if __name__ == '__main__':
main()
+103
View File
@@ -0,0 +1,103 @@
# -*- coding: utf-8 -*-
"""合并 0856 定向批(PDF + 页面)候选,核验后入库 y2027_tuimian_2027。
入库前必须挡掉三类假数据:
1. 校名其实是列名(那 4 个丢了中文前缀的旧文件会抽出「拟录取学院」当学校);
2. 名单类型不是推免(这批全部来自推免公示页,若被标成复试名单会污染推免占比);
3. 单专业接收人数超过该校该专业拟招总数(推免不可能吃掉比计划还多的人)。
"""
import csv
import json
import os
import re
import sqlite3
import time
ZX = r'D:/择校/数据库/zexiao_2027.db'
PDF_CAND = r'D:/择校/运行暂存/agent_ingest_20261101/cand_batch.json'
HTML_CAND = r'D:/择校/运行暂存/agent_ingest_20261101/cand_tm0856_html.json'
MANIFEST = r'D:/择校/运行暂存/agent_ingest_20261101/tm0856/fetch_manifest.csv'
OUT = r'D:/择校/运行暂存/agent_ingest_20261101/cand_tm0856_merged.json'
BAD_SCHOOL = re.compile(r'^(序号|拟录取学院|学院代码|专业代码|姓名|考生编号|拟录取|录取)')
def main():
man = {r['local']: r for r in csv.DictReader(open(MANIFEST, encoding='utf-8-sig'))
if r.get('local')}
valid_schools = {r['school'] for r in man.values()}
man = {r['local']: r for r in csv.DictReader(open(MANIFEST, encoding='utf-8-sig'))
if r.get('local')}
valid_schools = {r['school'] for r in man.values() if r.get('school')}
rows = []
for path in (PDF_CAND, HTML_CAND):
if not os.path.exists(path):
continue
for x in json.load(open(path, encoding='utf-8')):
# 只认清单里登过的原件,别按文件名前缀猜:前缀是中文校名,
# 用 startswith 过滤会把全部正常行挡掉、只留下校名已丢失的旧文件
if (x.get('local_file') or '') not in man:
continue
rows.append(x)
print('原始候选=%d' % len(rows))
bad = {}
out = []
for r in rows:
sch = (r.get('school') or '').strip()
if not sch or BAD_SCHOOL.match(sch) or sch not in valid_schools:
bad['school_invalid'] = bad.get('school_invalid', 0) + 1
continue
m = man.get(r.get('local_file') or '', {})
if m.get('kind') and m['kind'] != r.get('list_kind'):
r['list_kind'] = m['kind'] # 以清单登记的名单类型为准,防误标
# 解析器对名单型输出 person_count,对计数型输出 value,两个都要认:
# 只读 value 会把全部 PDF 行当成非法值拒掉
v = str(r.get('value') or r.get('person_count') or '').strip()
r['value'] = v
if not v.isdigit() or not (0 < int(v) <= 600):
bad['value_out_of_range'] = bad.get('value_out_of_range', 0) + 1
continue
if not (r.get('major_code') or r.get('major_name')):
bad['empty_major'] = bad.get('empty_major', 0) + 1
continue
page = m.get('page_url') or r.get('source_url') or ''
r['source_url'] = page
r['year'] = r.get('year') or m.get('year') or '2027'
r['plan_type'] = '普通推免'
r['target'] = 'tuimian'
r['note'] = (r.get('note') or '') + ';0856定向批'
out.append(r)
# 与该校该专业的拟招总数比对,超出的标 flag 不丢
z = sqlite3.connect(ZX, timeout=600)
z.execute('PRAGMA busy_timeout=600000')
quota = {}
for dwmc, zydm, s in z.execute("select dwmc,zydm,enroll_str from chsi_2027_dir "
"where trim(coalesce(enroll_str,''))<>''"):
m = re.search(r'(\d+)', s)
if m:
quota[(dwmc, zydm)] = int(m.group(1))
over = 0
for r in out:
q = quota.get((r['school'], r.get('major_code') or ''))
if q and int(r['value']) > q:
r['data_flag'] = 'exceeds_quota'
over += 1
seen = set()
ded = []
for r in out:
k = (r['school'], r.get('year'), r.get('college'), r.get('college_code'),
r.get('major_code'), r.get('major_name'), r.get('basis'))
if k in seen:
bad['dup_key'] = bad.get('dup_key', 0) + 1
continue
seen.add(k)
ded.append(r)
json.dump(ded, open(OUT, 'w', encoding='utf-8'), ensure_ascii=False, indent=1)
print('通过=%d 拒绝=%s 超拟招标flag=%d' % (len(ded), bad, over))
import collections
print('按校:', collections.Counter(r['school'] for r in ded).most_common())
print('按类型:', dict(collections.Counter(r['list_kind'] for r in ded)))
z.close()
if __name__ == '__main__':
main()
+145
View File
@@ -0,0 +1,145 @@
# -*- coding: utf-8 -*-
"""解析公示页 HTML 里直接排出的表格(无附件的那些学校)。
抓回来的 41 所报「页面无 PDF」,但其中不少是把名单或复试线直接排在页面表格里
(广东外语外贸大学那 95 行逐专业线就是这个形态)。
列语义一律按表头关键词定,不按位置认:同是「专业」列,有的写「专业代码」、
有的写「拟录取专业代码」,有的把代码与名称连写在一格。
"""
import csv
import glob
import html as htmllib
import json
import os
import re
RAW = r'D:/择校/运行暂存/agent_ingest_20261101/tm0856/raw'
MANIFEST = r'D:/择校/运行暂存/agent_ingest_20261101/tm0856/fetch_manifest.csv'
OUT = r'D:/择校/运行暂存/agent_ingest_20261101/cand_tm0856_html.json'
CODE_RE = re.compile(r'(?<![\dA-Za-z])(\d{6}|[0-9]{4}[A-Z][0-9]{1,2}|[0-9]{2}[A-Z][0-9]{3,4})(?![\dA-Za-z])')
HDR = re.compile(r'姓名|考生编号|证件号|报名号|专业代码|录取数|接收人数|复试线|总分|单科|专项计划')
def decode(b):
for e in ('utf-8', 'gbk', 'gb18030'):
try:
return b.decode(e)
except Exception:
continue
return b.decode('utf-8', 'ignore')
def cells(tr):
out = []
for c in re.findall(r'<t[dh][^>]*>(.*?)</t[dh]>', tr, re.S | re.I):
c = htmllib.unescape(re.sub(r'<[^>]+>', ' ', c)).replace('\xa0', ' ')
out.append(re.sub(r'\s+', ' ', c).strip())
return out
def header_map(hdr):
m = {}
for i, raw in enumerate(hdr):
c = raw.replace(' ', '')
if not c:
continue
if 'code' not in m and '专业代码' in c:
m['code'] = i
elif 'name' not in m and re.search(r'(专业|学科)(名称|全称|及名称)$|^(拟录取|录取)?(专业|学科)$|^学科专业$', c):
m['name'] = i
elif 'college' not in m and (re.search(r'^(拟录取|录取)?(学院|院系|院系所|单位)$', c)
or (re.search(r'(学院|院系|单位|研究院)', c) and re.search(r'(名称|全称)', c))):
m['college'] = i
elif 'college_code' not in m and re.search(r'(学院|院系所?|单位)(代码|码)$', c):
m['college_code'] = i
elif 'count' not in m and re.search(r'^(录取数|接收人数|拟接收|已接收人数)$', c):
m['count'] = i
elif 'total' not in m and re.search(r'总分|复试线$', c) and '要求' not in c:
m['total'] = i
elif 'l100' not in m and re.search(r'单科.*(=|=)?100', c) and '>' not in c:
m['l100'] = i
elif 'lover' not in m and re.search(r'单科.*>', c):
m['lover'] = i
elif 'person' not in m and re.search(r'^(考生)?姓名$', c):
m['person'] = i
return m
def main():
man = {}
if os.path.exists(MANIFEST):
for r in csv.DictReader(open(MANIFEST, encoding='utf-8-sig')):
man[r.get('local') or ''] = r
out = []
for p in sorted(glob.glob(RAW + '/*_page.html')):
fn = os.path.basename(p)
rec = man.get(fn, {})
school, year = rec.get('school', ''), rec.get('year', '')
if not school:
continue
t = decode(open(p, 'rb').read())
trs = re.findall(r'<tr[^>]*>(.*?)</tr>', t, re.S | re.I)
if len(trs) < 4:
print('%-22s %-12s 无表格行(%d)' % (school[:20], fn, len(trs)))
continue
hm, n_rows, n_line = None, 0, 0
agg = {}
for tr in trs:
cs = cells(tr)
if not any(cs):
continue
j = '|'.join(cs)
if HDR.search(j) and not CODE_RE.search(re.sub(r'(政治|外语|英语|总分|编号|成绩|学院|专业|姓名|线|要求|学习方式|专项|计划|备注|代码|名称|拟录取|录取|单科|满分|序号|层次|类型|码)', '', j)):
cand = header_map(cs)
if cand.get('code') is not None or cand.get('name') is not None or cand.get('count') is not None:
hm = cand
continue
if hm is None:
continue
if any(w in j for w in ('汇总', '合计', '小计')):
continue
def at(k):
i = hm.get(k)
return cs[i] if (i is not None and i < len(cs)) else ''
code = ''
cv = at('code')
m = CODE_RE.search((cv or '').replace(' ', ''))
if m:
code = m.group(1)
nm = at('name')
if not code and nm:
m2 = CODE_RE.search(nm.replace(' ', ''))
if m2:
code = m2.group(1)
nm = nm.split(code, 1)[-1]
cnt = at('count').strip()
tot = re.sub(r'[^\d]', '', at('total'))
if re.match(r'^\d{1,5}$', cnt) and (code or nm):
n_line += 1
out.append(dict(school=school, year=year, college=at('college'),
college_code=at('college_code'), major_code=code,
major_name=re.sub(r'^[))、,::\s]+', '', nm or ''),
kind='', study_mode='', value=cnt, basis='count_direct',
source_url=rec.get('page_url', ''), local_file=fn,
note='页面内表格的官方计数'))
elif (code or nm) and (at('person') or tot):
n_rows += 1
k = (school, year, at('college'), at('college_code'), code,
re.sub(r'^[))、,::\s]+', '', nm or ''))
agg[k] = agg.get(k, 0) + 1
for k, v in agg.items():
out.append(dict(school=k[0], year=k[1], college=k[2], college_code=k[3],
major_code=k[4], major_name=k[5], kind='', study_mode='',
value=str(v), basis='roster_derived',
source_url=rec.get('page_url', ''), local_file=fn,
note='页面内名单按专业分组行数'))
print('%-22s 计数行=%-4d 名单行=%-5d 分组键=%d' % (
school[:22], n_line, n_rows, len(agg)))
json.dump(out, open(OUT, 'w', encoding='utf-8'), ensure_ascii=False, indent=1)
print('合计候选=%d -> %s' % (len(out), OUT))
if __name__ == '__main__':
main()