134 lines
4.8 KiB
Python
134 lines
4.8 KiB
Python
#!/usr/bin/env python
|
|
# -*- coding: utf-8 -*-
|
|
"""
|
|
阶段一补充:用真实浏览器渲染页面,拿到 JS 动态生成的内容。
|
|
|
|
为什么需要:
|
|
博达系 CMS 的分数线正文常以「组件 + JS」方式注入(frag="窗口xx"),
|
|
纯 HTTP 取到的 HTML 里只有占位,正文是空的。渲染后表格/图片才会出现。
|
|
|
|
产物:
|
|
{院校}/render_XX.html 渲染后的 DOM
|
|
{院校}/shot_XX.png 正文区域截图(供 OCR)
|
|
D:/择校/原始来源/渲染清单.csv
|
|
|
|
约束:单页串行、每页间隔 2 秒、不拦截 JS、不改 UA 指纹、不注入规避脚本。
|
|
|
|
用法:
|
|
python -B stage_render.py --year 2026
|
|
python -B stage_render.py --url https://... --out D:/x/y
|
|
"""
|
|
import argparse
|
|
import csv
|
|
import os
|
|
import re
|
|
import sys
|
|
import time
|
|
|
|
MANIFEST = r"D:/择校/原始来源/下载清单.csv"
|
|
RENDER_MANIFEST = r"D:/择校/原始来源/渲染清单.csv"
|
|
GAP = 2.0
|
|
|
|
|
|
def article_html(page):
|
|
"""取正文容器 HTML;找不到就退回 body。"""
|
|
for sel in ["#vsb_content", ".v_news_content", "#content", ".article-content",
|
|
".wp_articlecontent", "article", "#vsb_content_2"]:
|
|
try:
|
|
el = page.query_selector(sel)
|
|
if el:
|
|
html = el.inner_html()
|
|
if len(html) > 200:
|
|
return el, html
|
|
except Exception:
|
|
continue
|
|
body = page.query_selector("body")
|
|
return body, (body.inner_html() if body else "")
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--year", default="")
|
|
ap.add_argument("--school", default="")
|
|
ap.add_argument("--url", default="")
|
|
ap.add_argument("--out", default="")
|
|
ap.add_argument("--limit", type=int, default=0)
|
|
a = ap.parse_args()
|
|
|
|
from playwright.sync_api import sync_playwright
|
|
|
|
targets = []
|
|
if a.url:
|
|
targets = [("手工", a.year or "2026", a.url, a.out or r"D:/择校/原始来源/网页快照/_manual")]
|
|
else:
|
|
rows = list(csv.DictReader(open(MANIFEST, encoding="utf-8-sig")))
|
|
rows = [r for r in rows if r["类型"] == "page" and not r["备注"].startswith("fail")]
|
|
if a.year:
|
|
rows = [r for r in rows if r["年份"] == a.year]
|
|
if a.school:
|
|
rows = [r for r in rows if r["院校"] == a.school]
|
|
# 同一院校只渲染「像分数线公告」的页面
|
|
rows = [r for r in rows if re.search(r"分数线|基本线|基本要求|复试", r["备注"] or "")]
|
|
seen = set()
|
|
for r in rows:
|
|
k = r["URL"]
|
|
if k in seen:
|
|
continue
|
|
seen.add(k)
|
|
d = os.path.dirname(r["本地路径"])
|
|
targets.append((r["院校"], r["年份"], r["URL"], d))
|
|
if a.limit:
|
|
targets = targets[:a.limit]
|
|
|
|
print(f"待渲染 {len(targets)} 页")
|
|
if not targets:
|
|
return
|
|
|
|
out_rows = []
|
|
with sync_playwright() as pw:
|
|
browser = pw.chromium.launch(headless=True)
|
|
ctx = browser.new_context(viewport={"width": 1440, "height": 1200},
|
|
locale="zh-CN")
|
|
page = ctx.new_page()
|
|
for i, (school, year, url, d) in enumerate(targets, 1):
|
|
os.makedirs(d, exist_ok=True)
|
|
try:
|
|
page.goto(url, wait_until="networkidle", timeout=45000)
|
|
page.wait_for_timeout(1200)
|
|
except Exception as e:
|
|
print(f"[{i}/{len(targets)}] {school} 打开失败 {type(e).__name__}")
|
|
out_rows.append([school, year, url, "", "", 0, f"fail:{type(e).__name__}"])
|
|
time.sleep(GAP)
|
|
continue
|
|
el, html = article_html(page)
|
|
nt = len(re.findall(r"<table", html, re.I))
|
|
ni = len(re.findall(r"<img", html, re.I))
|
|
hp = os.path.join(d, f"render_{i:02d}.html")
|
|
with open(hp, "w", encoding="utf-8") as f:
|
|
f.write(html)
|
|
sp = os.path.join(d, f"shot_{i:02d}.png")
|
|
shot = ""
|
|
if nt == 0:
|
|
try:
|
|
el.screenshot(path=sp)
|
|
shot = sp
|
|
except Exception:
|
|
shot = ""
|
|
out_rows.append([school, year, url, hp, shot, nt, f"tables={nt};imgs={ni}"])
|
|
print(f"[{i}/{len(targets)}] {school:14s} 表格{nt} 图{ni} "
|
|
f"{'截图' if shot else ''}")
|
|
time.sleep(GAP)
|
|
ctx.close()
|
|
browser.close()
|
|
|
|
os.makedirs(os.path.dirname(RENDER_MANIFEST), exist_ok=True)
|
|
with open(RENDER_MANIFEST, "w", newline="", encoding="utf-8-sig") as f:
|
|
w = csv.writer(f)
|
|
w.writerow(["院校", "年份", "URL", "渲染HTML", "截图", "表格数", "备注"])
|
|
w.writerows(out_rows)
|
|
print(f"\n写出 {RENDER_MANIFEST}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|