Files

134 lines
4.8 KiB
Python

#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
阶段一补充:用真实浏览器渲染页面,拿到 JS 动态生成的内容。
为什么需要:
博达系 CMS 的分数线正文常以「组件 + JS」方式注入(frag="窗口xx"),
纯 HTTP 取到的 HTML 里只有占位,正文是空的。渲染后表格/图片才会出现。
产物:
{院校}/render_XX.html 渲染后的 DOM
{院校}/shot_XX.png 正文区域截图(供 OCR)
D:/择校/原始来源/渲染清单.csv
约束:单页串行、每页间隔 2 秒、不拦截 JS、不改 UA 指纹、不注入规避脚本。
用法:
python -B stage_render.py --year 2026
python -B stage_render.py --url https://... --out D:/x/y
"""
import argparse
import csv
import os
import re
import sys
import time
MANIFEST = r"D:/择校/原始来源/下载清单.csv"
RENDER_MANIFEST = r"D:/择校/原始来源/渲染清单.csv"
GAP = 2.0
def article_html(page):
"""取正文容器 HTML;找不到就退回 body。"""
for sel in ["#vsb_content", ".v_news_content", "#content", ".article-content",
".wp_articlecontent", "article", "#vsb_content_2"]:
try:
el = page.query_selector(sel)
if el:
html = el.inner_html()
if len(html) > 200:
return el, html
except Exception:
continue
body = page.query_selector("body")
return body, (body.inner_html() if body else "")
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--year", default="")
ap.add_argument("--school", default="")
ap.add_argument("--url", default="")
ap.add_argument("--out", default="")
ap.add_argument("--limit", type=int, default=0)
a = ap.parse_args()
from playwright.sync_api import sync_playwright
targets = []
if a.url:
targets = [("手工", a.year or "2026", a.url, a.out or r"D:/择校/原始来源/网页快照/_manual")]
else:
rows = list(csv.DictReader(open(MANIFEST, encoding="utf-8-sig")))
rows = [r for r in rows if r["类型"] == "page" and not r["备注"].startswith("fail")]
if a.year:
rows = [r for r in rows if r["年份"] == a.year]
if a.school:
rows = [r for r in rows if r["院校"] == a.school]
# 同一院校只渲染「像分数线公告」的页面
rows = [r for r in rows if re.search(r"分数线|基本线|基本要求|复试", r["备注"] or "")]
seen = set()
for r in rows:
k = r["URL"]
if k in seen:
continue
seen.add(k)
d = os.path.dirname(r["本地路径"])
targets.append((r["院校"], r["年份"], r["URL"], d))
if a.limit:
targets = targets[:a.limit]
print(f"待渲染 {len(targets)} 页")
if not targets:
return
out_rows = []
with sync_playwright() as pw:
browser = pw.chromium.launch(headless=True)
ctx = browser.new_context(viewport={"width": 1440, "height": 1200},
locale="zh-CN")
page = ctx.new_page()
for i, (school, year, url, d) in enumerate(targets, 1):
os.makedirs(d, exist_ok=True)
try:
page.goto(url, wait_until="networkidle", timeout=45000)
page.wait_for_timeout(1200)
except Exception as e:
print(f"[{i}/{len(targets)}] {school} 打开失败 {type(e).__name__}")
out_rows.append([school, year, url, "", "", 0, f"fail:{type(e).__name__}"])
time.sleep(GAP)
continue
el, html = article_html(page)
nt = len(re.findall(r"<table", html, re.I))
ni = len(re.findall(r"<img", html, re.I))
hp = os.path.join(d, f"render_{i:02d}.html")
with open(hp, "w", encoding="utf-8") as f:
f.write(html)
sp = os.path.join(d, f"shot_{i:02d}.png")
shot = ""
if nt == 0:
try:
el.screenshot(path=sp)
shot = sp
except Exception:
shot = ""
out_rows.append([school, year, url, hp, shot, nt, f"tables={nt};imgs={ni}"])
print(f"[{i}/{len(targets)}] {school:14s} 表格{nt} 图{ni} "
f"{'截图' if shot else ''}")
time.sleep(GAP)
ctx.close()
browser.close()
os.makedirs(os.path.dirname(RENDER_MANIFEST), exist_ok=True)
with open(RENDER_MANIFEST, "w", newline="", encoding="utf-8-sig") as f:
w = csv.writer(f)
w.writerow(["院校", "年份", "URL", "渲染HTML", "截图", "表格数", "备注"])
w.writerows(out_rows)
print(f"\n写出 {RENDER_MANIFEST}")
if __name__ == "__main__":
main()