补齐0856定向批的抓取与页面表格解析链
三个判据太窄的地方一起放开:附件不止 PDF(还有 xls/docx/zip), 表格可能直接排在页面 HTML 里,个别院校站证书链不全要能降级重试并留痕。 第一版因此把 41 所报成「页面无 PDF」,那是判据问题不是学校没数据。 另修文件名 bug:用 [^0-9A-Za-z] 清洗中文校名会把名字剥成空串, 落盘全成 _0_xxx.pdf,回查时认不出学校;改按 Windows 非法字符过滤并保留中文。
This commit is contained in:
1 parent
b0c289b797
commit
f7b95e5e7f
4 files changed
+530
No files matched your search
@@ -0,0 +1,44 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""把抓取结果与探测清单合成解析器认得的 manifest。
|
||||
|
||||
解析器按 local 文件名回查校名/年份/来源,而名单类型只能从标题判断
|
||||
(URL 常是 /info/1004/9004.htm 这种无语义路径),所以要把 roster_title 带进来。
|
||||
"""
|
||||
import csv
|
||||
import os
|
||||
|
||||
BASE = r'D:/择校/运行暂存/agent_ingest_20261101/tm0856'
|
||||
PROBE = r'D:/择校/运行暂存/agent_search_20261101/tuimian_0856/probe_full.csv'
|
||||
OUT = os.path.join(BASE, 'fetch_manifest_enriched.csv')
|
||||
|
||||
|
||||
def main():
|
||||
by_school = {}
|
||||
for r in csv.DictReader(open(PROBE, encoding='utf-8-sig')):
|
||||
by_school.setdefault(r['school'], r)
|
||||
rows = list(csv.DictReader(open(os.path.join(BASE, 'fetch_manifest.csv'), encoding='utf-8-sig')))
|
||||
out = []
|
||||
for r in rows:
|
||||
if r['status'] != 'ok' or not (r.get('local') or '').strip():
|
||||
continue
|
||||
p = by_school.get(r['school'], {})
|
||||
title = (p.get('roster_title') or '') + ' ' + (p.get('roster_url') or '')
|
||||
kind = '推免拟录取名单' if any(k in title for k in ('推免', '免试', '免推', 'tms', 'tmnlq')) \
|
||||
else '复试名单'
|
||||
out.append(dict(school=r['school'], year=r['year'], kind=kind,
|
||||
local=r['local'], url=r['pdf_url'], page_url=r['page_url'],
|
||||
status='ok', bytes=r['bytes'], pages=r['pages'], rows=r['rows'],
|
||||
sha256='', content_type='application/pdf', error=''))
|
||||
cols = list(out[0].keys()) if out else []
|
||||
with open(OUT, 'w', newline='', encoding='utf-8-sig') as f:
|
||||
w = csv.DictWriter(f, fieldnames=cols)
|
||||
w.writeheader()
|
||||
w.writerows(out)
|
||||
import collections
|
||||
print('可用=%d 校=%d' % (len(out), len({r['school'] for r in out})))
|
||||
print('名单类型:', dict(collections.Counter(r['kind'] for r in out)))
|
||||
print('写入', OUT)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,238 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""按 agent 探到的 67 个 2027 推免公示页抓数据:页面 -> 附件 -> 表格缓存。
|
||||
|
||||
第一版只认 .pdf 附件,结果 41 所报「页面无 PDF」——那是判据太窄,不是学校没数据。
|
||||
名单可能在这几种载体里,逐个都要认:
|
||||
1. 附件是 xls/xlsx/docx/zip/rar 而非 pdf;
|
||||
2. 表格直接排在页面 HTML 里(广外那 95 行逐专业线就是这么来的);
|
||||
3. 站点证书链不全,裸请求报 SSLError(单独降级重试并留痕,不因此判该校无数据)。
|
||||
名单年份不是 2027 的一律不收:混进 2027 列会直接算错推免占比。
|
||||
文件名保留中文校名,只替换 Windows 非法字符——上一版用 [^0-9A-Za-z] 过滤,
|
||||
中文校名被剥成空串,落盘全成了 _0_xxx.pdf,回查时对不上学校。
|
||||
磁盘预算 300MB,单文件上限 8MB,每校最多 3 份附件。
|
||||
"""
|
||||
import csv
|
||||
import html as htmllib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import threading
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
import fitz
|
||||
import requests
|
||||
import urllib3
|
||||
|
||||
SRC = r'D:/择校/运行暂存/agent_search_20261101/tuimian_0856/probe_full.csv'
|
||||
BASE = r'D:/择校/运行暂存/agent_ingest_20261101/tm0856'
|
||||
RAW = os.path.join(BASE, 'raw')
|
||||
CACHE = os.path.join(BASE, 'grid_cache')
|
||||
MANIFEST = os.path.join(BASE, 'fetch_manifest.csv')
|
||||
UA = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 '
|
||||
'Chrome/120 Safari/537.36'}
|
||||
MAX_FILE = 8 * 1024 * 1024
|
||||
BUDGET = [300 * 1024 * 1024]
|
||||
ATTACH_EXT = r'(?:pdf|xls|xlsx|doc|docx|zip|rar|wps)'
|
||||
lock = threading.Lock()
|
||||
OLE = bytes([0xD0, 0xCF, 0x11, 0xE0])
|
||||
ZIP2 = (bytes([0x50, 0x4B, 0x03, 0x04]), bytes([0x50, 0x4B, 0x05, 0x06]))
|
||||
|
||||
|
||||
def decode(b):
|
||||
for e in ('utf-8', 'gbk', 'gb18030'):
|
||||
try:
|
||||
return b.decode(e)
|
||||
except Exception:
|
||||
continue
|
||||
return b.decode('utf-8', 'ignore')
|
||||
|
||||
|
||||
def safe(s):
|
||||
return re.sub(r'[\\/:*?"<>|\s]+', '', s)[:14] or 'x'
|
||||
|
||||
|
||||
def attachment_links(page_url, body):
|
||||
res, seen = [], set()
|
||||
scheme, rest = page_url.split('://', 1)
|
||||
host = rest.split('/', 1)[0]
|
||||
for m in re.finditer(r'href=["\']([^"\']+)["\']', body, re.I):
|
||||
u = htmllib.unescape(m.group(1).strip())
|
||||
if not u or u.lower().startswith(('javascript:', 'mailto:', '#', 'data:')):
|
||||
continue
|
||||
if not re.search(r'\.' + ATTACH_EXT + r'($|\?)', u, re.I):
|
||||
continue
|
||||
if u.startswith('//'):
|
||||
u = scheme + ':' + u
|
||||
elif u.startswith('/'):
|
||||
u = '%s://%s%s' % (scheme, host, u)
|
||||
elif not u.startswith('http'):
|
||||
u = page_url.rsplit('/', 1)[0] + '/' + u
|
||||
if u not in seen:
|
||||
seen.add(u)
|
||||
res.append(u)
|
||||
return res
|
||||
|
||||
|
||||
def rows_from_words(page, ytol=4.0):
|
||||
"""无框线表格的兜底重建:按 y 聚类成行、x 排序成列。"""
|
||||
ws = page.get_text('words')
|
||||
if not ws:
|
||||
return []
|
||||
ws.sort(key=lambda w: (w[1], w[0]))
|
||||
rows, cur, ly = [], [], None
|
||||
for w in ws:
|
||||
y = (w[1] + w[3]) / 2.0
|
||||
if ly is None or abs(y - ly) <= ytol:
|
||||
cur.append(w)
|
||||
ly = y if ly is None else (ly + y) / 2.0
|
||||
else:
|
||||
rows.append(cur)
|
||||
cur, ly = [w], y
|
||||
if cur:
|
||||
rows.append(cur)
|
||||
return [[w[4] for w in sorted(r, key=lambda w: w[0])] for r in rows]
|
||||
|
||||
|
||||
def grid_of_pdf(path):
|
||||
doc = fitz.open(path)
|
||||
pages = []
|
||||
for page in doc:
|
||||
try:
|
||||
tabs = list(page.find_tables().tables)
|
||||
except Exception:
|
||||
tabs = []
|
||||
g = []
|
||||
for tb in tabs:
|
||||
try:
|
||||
for row in tb.extract():
|
||||
g.append([('' if c is None else str(c)) for c in row])
|
||||
except Exception:
|
||||
continue
|
||||
rebuilt = rows_from_words(page)
|
||||
n_re = sum(1 for r in rebuilt if any(str(c).strip() for c in r))
|
||||
if n_re > sum(1 for r in g if any(str(c).strip() for c in r)):
|
||||
g = rebuilt # 靠排版对齐画的表,find_tables 只会给一排空格
|
||||
pages.append(g)
|
||||
n = doc.page_count
|
||||
doc.close()
|
||||
return pages, n
|
||||
|
||||
|
||||
def get(url, referer=None, verify=True):
|
||||
h = dict(UA)
|
||||
if referer:
|
||||
h['Referer'] = referer
|
||||
return requests.get(url, headers=h, timeout=60, stream=True, verify=verify)
|
||||
|
||||
|
||||
def work(row):
|
||||
school, page_url = row['school'], row['roster_url']
|
||||
year = row.get('roster_year') or ''
|
||||
rec = dict(school=school, year=year, page_url=page_url, pdf_url='', local='',
|
||||
status='', bytes=0, pages=0, rows=0, note='')
|
||||
if year != '2027':
|
||||
rec['status'] = 'wrong_year_%s' % year
|
||||
return rec
|
||||
try:
|
||||
try:
|
||||
pr = get(page_url)
|
||||
except requests.exceptions.SSLError:
|
||||
pr = get(page_url, verify=False)
|
||||
rec['note'] = '证书链不全已降级重试'
|
||||
if pr.status_code != 200:
|
||||
rec['status'] = 'page_http_%s' % pr.status_code
|
||||
return rec
|
||||
body = decode(pr.content)
|
||||
links = attachment_links(page_url, body)
|
||||
if not links:
|
||||
name = safe(school) + '_page.html'
|
||||
open(os.path.join(RAW, name), 'wb').write(pr.content)
|
||||
rec.update(local=name, status='page_only', bytes=len(pr.content))
|
||||
print('pg %-16s 页面已存(无附件,待 HTML 解析)' % school[:16])
|
||||
return rec
|
||||
for i, u in enumerate(links[:3]):
|
||||
with lock:
|
||||
if BUDGET[0] <= 0:
|
||||
rec['status'] = 'disk_budget'
|
||||
return rec
|
||||
try:
|
||||
rr = get(u, referer=page_url)
|
||||
except requests.exceptions.SSLError:
|
||||
rr = get(u, referer=page_url, verify=False)
|
||||
if rr.status_code != 200:
|
||||
continue
|
||||
ext = re.split(r'[?#]', u.rsplit('.', 1)[-1])[0].lower()
|
||||
if ext not in ('pdf', 'xls', 'xlsx', 'doc', 'docx', 'zip', 'rar', 'wps'):
|
||||
ext = 'bin'
|
||||
name = '%s_%d_%s.%s' % (safe(school), i, re.sub(r'\W', '', u)[-8:], ext)
|
||||
dst = os.path.join(RAW, name)
|
||||
tot = 0
|
||||
with open(dst, 'wb') as f:
|
||||
for ch in rr.iter_content(65536):
|
||||
tot += len(ch)
|
||||
if tot > MAX_FILE:
|
||||
break
|
||||
f.write(ch)
|
||||
head = open(dst, 'rb').read(8)
|
||||
if not (b'%PDF' in head or head[:4] == OLE or head[:4] in ZIP2
|
||||
or head[:3] == b'Rar'):
|
||||
os.remove(dst) # 端点回的多半是登录页或 HTML 跳转
|
||||
continue
|
||||
with lock:
|
||||
BUDGET[0] -= tot
|
||||
if b'%PDF' not in head:
|
||||
rec.update(pdf_url=u, local=name, status='attachment_nonpdf', bytes=tot,
|
||||
note=(rec['note'] + ';非PDF附件待解析').strip(';'))
|
||||
print('att %-16s %s %.1fMB 非PDF,已留档' % (school[:16], ext, tot / 2 ** 20))
|
||||
return rec
|
||||
pages, npg = grid_of_pdf(dst)
|
||||
nb = sum(1 for g in pages for x in g if any(str(c).strip() for c in x))
|
||||
json.dump({'file': name, 'pages': pages},
|
||||
open(os.path.join(CACHE, name.rsplit('.', 1)[0] + '.json'),
|
||||
'w', encoding='utf-8'), ensure_ascii=False)
|
||||
rec.update(pdf_url=u, local=name, status='ok' if nb > 5 else 'no_table_rows',
|
||||
bytes=tot, pages=npg, rows=nb)
|
||||
print('ok %-16s pages=%-4d rows=%-5d %.1fMB' % (
|
||||
school[:16], npg, nb, tot / 2 ** 20))
|
||||
return rec
|
||||
rec['status'] = 'all_attachments_failed'
|
||||
except Exception as e:
|
||||
rec['status'] = 'err:%s' % type(e).__name__
|
||||
print('-- %-16s %s' % (school[:16], rec['status']))
|
||||
return rec
|
||||
|
||||
|
||||
def main():
|
||||
urllib3.disable_warnings()
|
||||
os.makedirs(RAW, exist_ok=True)
|
||||
os.makedirs(CACHE, exist_ok=True)
|
||||
done = set()
|
||||
if os.path.exists(MANIFEST):
|
||||
for r in csv.DictReader(open(MANIFEST, encoding='utf-8-sig')):
|
||||
if r.get('status') == 'ok':
|
||||
done.add(r['school'])
|
||||
rows = [r for r in csv.DictReader(open(SRC, encoding='utf-8-sig'))
|
||||
if r['verdict'] == 'roster_found' and r['school'] not in done]
|
||||
print('待取 %d 校(跳过已成功 %d)' % (len(rows), len(done)))
|
||||
with ThreadPoolExecutor(max_workers=6) as ex:
|
||||
out = list(ex.map(work, rows))
|
||||
old = []
|
||||
if os.path.exists(MANIFEST):
|
||||
old = [r for r in csv.DictReader(open(MANIFEST, encoding='utf-8-sig'))]
|
||||
allrows = old + out
|
||||
cols = ['school', 'year', 'page_url', 'pdf_url', 'local', 'status', 'bytes',
|
||||
'pages', 'rows', 'note']
|
||||
with open(MANIFEST, 'w', newline='', encoding='utf-8-sig') as f:
|
||||
w = csv.DictWriter(f, fieldnames=cols, extrasaction='ignore', restval='')
|
||||
w.writeheader()
|
||||
w.writerows(allrows)
|
||||
import collections
|
||||
okn = [x for x in allrows if x['status'] == 'ok']
|
||||
print('本批成功=%d 累计成功=%d 校 累计表格行=%d' % (
|
||||
sum(1 for x in out if x['status'] == 'ok'), len(okn),
|
||||
sum(int(x.get('rows') or 0) for x in okn)))
|
||||
print('形态:', dict(collections.Counter(x['status'] for x in allrows)))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,103 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""合并 0856 定向批(PDF + 页面)候选,核验后入库 y2027_tuimian_2027。
|
||||
|
||||
入库前必须挡掉三类假数据:
|
||||
1. 校名其实是列名(那 4 个丢了中文前缀的旧文件会抽出「拟录取学院」当学校);
|
||||
2. 名单类型不是推免(这批全部来自推免公示页,若被标成复试名单会污染推免占比);
|
||||
3. 单专业接收人数超过该校该专业拟招总数(推免不可能吃掉比计划还多的人)。
|
||||
"""
|
||||
import csv
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sqlite3
|
||||
import time
|
||||
|
||||
ZX = r'D:/择校/数据库/zexiao_2027.db'
|
||||
PDF_CAND = r'D:/择校/运行暂存/agent_ingest_20261101/cand_batch.json'
|
||||
HTML_CAND = r'D:/择校/运行暂存/agent_ingest_20261101/cand_tm0856_html.json'
|
||||
MANIFEST = r'D:/择校/运行暂存/agent_ingest_20261101/tm0856/fetch_manifest.csv'
|
||||
OUT = r'D:/择校/运行暂存/agent_ingest_20261101/cand_tm0856_merged.json'
|
||||
BAD_SCHOOL = re.compile(r'^(序号|拟录取学院|学院代码|专业代码|姓名|考生编号|拟录取|录取)')
|
||||
|
||||
|
||||
def main():
|
||||
man = {r['local']: r for r in csv.DictReader(open(MANIFEST, encoding='utf-8-sig'))
|
||||
if r.get('local')}
|
||||
valid_schools = {r['school'] for r in man.values()}
|
||||
man = {r['local']: r for r in csv.DictReader(open(MANIFEST, encoding='utf-8-sig'))
|
||||
if r.get('local')}
|
||||
valid_schools = {r['school'] for r in man.values() if r.get('school')}
|
||||
rows = []
|
||||
for path in (PDF_CAND, HTML_CAND):
|
||||
if not os.path.exists(path):
|
||||
continue
|
||||
for x in json.load(open(path, encoding='utf-8')):
|
||||
# 只认清单里登过的原件,别按文件名前缀猜:前缀是中文校名,
|
||||
# 用 startswith 过滤会把全部正常行挡掉、只留下校名已丢失的旧文件
|
||||
if (x.get('local_file') or '') not in man:
|
||||
continue
|
||||
rows.append(x)
|
||||
print('原始候选=%d' % len(rows))
|
||||
bad = {}
|
||||
out = []
|
||||
for r in rows:
|
||||
sch = (r.get('school') or '').strip()
|
||||
if not sch or BAD_SCHOOL.match(sch) or sch not in valid_schools:
|
||||
bad['school_invalid'] = bad.get('school_invalid', 0) + 1
|
||||
continue
|
||||
m = man.get(r.get('local_file') or '', {})
|
||||
if m.get('kind') and m['kind'] != r.get('list_kind'):
|
||||
r['list_kind'] = m['kind'] # 以清单登记的名单类型为准,防误标
|
||||
# 解析器对名单型输出 person_count,对计数型输出 value,两个都要认:
|
||||
# 只读 value 会把全部 PDF 行当成非法值拒掉
|
||||
v = str(r.get('value') or r.get('person_count') or '').strip()
|
||||
r['value'] = v
|
||||
if not v.isdigit() or not (0 < int(v) <= 600):
|
||||
bad['value_out_of_range'] = bad.get('value_out_of_range', 0) + 1
|
||||
continue
|
||||
if not (r.get('major_code') or r.get('major_name')):
|
||||
bad['empty_major'] = bad.get('empty_major', 0) + 1
|
||||
continue
|
||||
page = m.get('page_url') or r.get('source_url') or ''
|
||||
r['source_url'] = page
|
||||
r['year'] = r.get('year') or m.get('year') or '2027'
|
||||
r['plan_type'] = '普通推免'
|
||||
r['target'] = 'tuimian'
|
||||
r['note'] = (r.get('note') or '') + ';0856定向批'
|
||||
out.append(r)
|
||||
# 与该校该专业的拟招总数比对,超出的标 flag 不丢
|
||||
z = sqlite3.connect(ZX, timeout=600)
|
||||
z.execute('PRAGMA busy_timeout=600000')
|
||||
quota = {}
|
||||
for dwmc, zydm, s in z.execute("select dwmc,zydm,enroll_str from chsi_2027_dir "
|
||||
"where trim(coalesce(enroll_str,''))<>''"):
|
||||
m = re.search(r'(\d+)', s)
|
||||
if m:
|
||||
quota[(dwmc, zydm)] = int(m.group(1))
|
||||
over = 0
|
||||
for r in out:
|
||||
q = quota.get((r['school'], r.get('major_code') or ''))
|
||||
if q and int(r['value']) > q:
|
||||
r['data_flag'] = 'exceeds_quota'
|
||||
over += 1
|
||||
seen = set()
|
||||
ded = []
|
||||
for r in out:
|
||||
k = (r['school'], r.get('year'), r.get('college'), r.get('college_code'),
|
||||
r.get('major_code'), r.get('major_name'), r.get('basis'))
|
||||
if k in seen:
|
||||
bad['dup_key'] = bad.get('dup_key', 0) + 1
|
||||
continue
|
||||
seen.add(k)
|
||||
ded.append(r)
|
||||
json.dump(ded, open(OUT, 'w', encoding='utf-8'), ensure_ascii=False, indent=1)
|
||||
print('通过=%d 拒绝=%s 超拟招标flag=%d' % (len(ded), bad, over))
|
||||
import collections
|
||||
print('按校:', collections.Counter(r['school'] for r in ded).most_common())
|
||||
print('按类型:', dict(collections.Counter(r['list_kind'] for r in ded)))
|
||||
z.close()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,145 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""解析公示页 HTML 里直接排出的表格(无附件的那些学校)。
|
||||
|
||||
抓回来的 41 所报「页面无 PDF」,但其中不少是把名单或复试线直接排在页面表格里
|
||||
(广东外语外贸大学那 95 行逐专业线就是这个形态)。
|
||||
列语义一律按表头关键词定,不按位置认:同是「专业」列,有的写「专业代码」、
|
||||
有的写「拟录取专业代码」,有的把代码与名称连写在一格。
|
||||
"""
|
||||
import csv
|
||||
import glob
|
||||
import html as htmllib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
|
||||
RAW = r'D:/择校/运行暂存/agent_ingest_20261101/tm0856/raw'
|
||||
MANIFEST = r'D:/择校/运行暂存/agent_ingest_20261101/tm0856/fetch_manifest.csv'
|
||||
OUT = r'D:/择校/运行暂存/agent_ingest_20261101/cand_tm0856_html.json'
|
||||
|
||||
CODE_RE = re.compile(r'(?<![\dA-Za-z])(\d{6}|[0-9]{4}[A-Z][0-9]{1,2}|[0-9]{2}[A-Z][0-9]{3,4})(?![\dA-Za-z])')
|
||||
HDR = re.compile(r'姓名|考生编号|证件号|报名号|专业代码|录取数|接收人数|复试线|总分|单科|专项计划')
|
||||
|
||||
|
||||
def decode(b):
|
||||
for e in ('utf-8', 'gbk', 'gb18030'):
|
||||
try:
|
||||
return b.decode(e)
|
||||
except Exception:
|
||||
continue
|
||||
return b.decode('utf-8', 'ignore')
|
||||
|
||||
|
||||
def cells(tr):
|
||||
out = []
|
||||
for c in re.findall(r'<t[dh][^>]*>(.*?)</t[dh]>', tr, re.S | re.I):
|
||||
c = htmllib.unescape(re.sub(r'<[^>]+>', ' ', c)).replace('\xa0', ' ')
|
||||
out.append(re.sub(r'\s+', ' ', c).strip())
|
||||
return out
|
||||
|
||||
|
||||
def header_map(hdr):
|
||||
m = {}
|
||||
for i, raw in enumerate(hdr):
|
||||
c = raw.replace(' ', '')
|
||||
if not c:
|
||||
continue
|
||||
if 'code' not in m and '专业代码' in c:
|
||||
m['code'] = i
|
||||
elif 'name' not in m and re.search(r'(专业|学科)(名称|全称|及名称)$|^(拟录取|录取)?(专业|学科)$|^学科专业$', c):
|
||||
m['name'] = i
|
||||
elif 'college' not in m and (re.search(r'^(拟录取|录取)?(学院|院系|院系所|单位)$', c)
|
||||
or (re.search(r'(学院|院系|单位|研究院)', c) and re.search(r'(名称|全称)', c))):
|
||||
m['college'] = i
|
||||
elif 'college_code' not in m and re.search(r'(学院|院系所?|单位)(代码|码)$', c):
|
||||
m['college_code'] = i
|
||||
elif 'count' not in m and re.search(r'^(录取数|接收人数|拟接收|已接收人数)$', c):
|
||||
m['count'] = i
|
||||
elif 'total' not in m and re.search(r'总分|复试线$', c) and '要求' not in c:
|
||||
m['total'] = i
|
||||
elif 'l100' not in m and re.search(r'单科.*(=|=)?100', c) and '>' not in c:
|
||||
m['l100'] = i
|
||||
elif 'lover' not in m and re.search(r'单科.*>', c):
|
||||
m['lover'] = i
|
||||
elif 'person' not in m and re.search(r'^(考生)?姓名$', c):
|
||||
m['person'] = i
|
||||
return m
|
||||
|
||||
|
||||
def main():
|
||||
man = {}
|
||||
if os.path.exists(MANIFEST):
|
||||
for r in csv.DictReader(open(MANIFEST, encoding='utf-8-sig')):
|
||||
man[r.get('local') or ''] = r
|
||||
out = []
|
||||
for p in sorted(glob.glob(RAW + '/*_page.html')):
|
||||
fn = os.path.basename(p)
|
||||
rec = man.get(fn, {})
|
||||
school, year = rec.get('school', ''), rec.get('year', '')
|
||||
if not school:
|
||||
continue
|
||||
t = decode(open(p, 'rb').read())
|
||||
trs = re.findall(r'<tr[^>]*>(.*?)</tr>', t, re.S | re.I)
|
||||
if len(trs) < 4:
|
||||
print('%-22s %-12s 无表格行(%d)' % (school[:20], fn, len(trs)))
|
||||
continue
|
||||
hm, n_rows, n_line = None, 0, 0
|
||||
agg = {}
|
||||
for tr in trs:
|
||||
cs = cells(tr)
|
||||
if not any(cs):
|
||||
continue
|
||||
j = '|'.join(cs)
|
||||
if HDR.search(j) and not CODE_RE.search(re.sub(r'(政治|外语|英语|总分|编号|成绩|学院|专业|姓名|线|要求|学习方式|专项|计划|备注|代码|名称|拟录取|录取|单科|满分|序号|层次|类型|码)', '', j)):
|
||||
cand = header_map(cs)
|
||||
if cand.get('code') is not None or cand.get('name') is not None or cand.get('count') is not None:
|
||||
hm = cand
|
||||
continue
|
||||
if hm is None:
|
||||
continue
|
||||
if any(w in j for w in ('汇总', '合计', '小计')):
|
||||
continue
|
||||
|
||||
def at(k):
|
||||
i = hm.get(k)
|
||||
return cs[i] if (i is not None and i < len(cs)) else ''
|
||||
code = ''
|
||||
cv = at('code')
|
||||
m = CODE_RE.search((cv or '').replace(' ', ''))
|
||||
if m:
|
||||
code = m.group(1)
|
||||
nm = at('name')
|
||||
if not code and nm:
|
||||
m2 = CODE_RE.search(nm.replace(' ', ''))
|
||||
if m2:
|
||||
code = m2.group(1)
|
||||
nm = nm.split(code, 1)[-1]
|
||||
cnt = at('count').strip()
|
||||
tot = re.sub(r'[^\d]', '', at('total'))
|
||||
if re.match(r'^\d{1,5}$', cnt) and (code or nm):
|
||||
n_line += 1
|
||||
out.append(dict(school=school, year=year, college=at('college'),
|
||||
college_code=at('college_code'), major_code=code,
|
||||
major_name=re.sub(r'^[))、,::\s]+', '', nm or ''),
|
||||
kind='', study_mode='', value=cnt, basis='count_direct',
|
||||
source_url=rec.get('page_url', ''), local_file=fn,
|
||||
note='页面内表格的官方计数'))
|
||||
elif (code or nm) and (at('person') or tot):
|
||||
n_rows += 1
|
||||
k = (school, year, at('college'), at('college_code'), code,
|
||||
re.sub(r'^[))、,::\s]+', '', nm or ''))
|
||||
agg[k] = agg.get(k, 0) + 1
|
||||
for k, v in agg.items():
|
||||
out.append(dict(school=k[0], year=k[1], college=k[2], college_code=k[3],
|
||||
major_code=k[4], major_name=k[5], kind='', study_mode='',
|
||||
value=str(v), basis='roster_derived',
|
||||
source_url=rec.get('page_url', ''), local_file=fn,
|
||||
note='页面内名单按专业分组行数'))
|
||||
print('%-22s 计数行=%-4d 名单行=%-5d 分组键=%d' % (
|
||||
school[:22], n_line, n_rows, len(agg)))
|
||||
json.dump(out, open(OUT, 'w', encoding='utf-8'), ensure_ascii=False, indent=1)
|
||||
print('合计候选=%d -> %s' % (len(out), OUT))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in new issue
Block a user