Files

143 lines
6.2 KiB
Python
Raw Permalink Blame History

This file contains invisible Unicode characters
This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
# -*- coding: utf-8 -*-
"""全库文本编码体检:找 mojibake、替换符、不可见字符、控制字符、括号全半角混用。
动机:对齐时发现校名「中国石油大学(北京)」官方半角括号、平台全角括号,
52 行因此没匹配上。同类问题会静默压低任何按名称关联的命中率,所以要一次扫干净。
只报告不改数据;要改的进 errata。
"""
import os
import re
import sqlite3
import sys
import unicodedata
from collections import Counter, defaultdict
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
HERE = os.path.dirname(os.path.abspath(__file__))
DB = os.path.join(HERE, "zexiao_2027.db")
# mojibake 判据收紧:单看字符形状会误伤(「〔2013〕19号」是公文正常括号,不是乱码)。
# 真乱码的特征是「按 latin-1 重编回去、再用 gbk 能解出中文」,且串里拉丁扩展字符成串出现。
LATIN_RUN = re.compile(r"[À-ÿ]{2,}")
MOJI = re.compile(r"(锟斤拷|拷拷|â€|Ã.)")
def is_mojibake(s):
if not LATIN_RUN.search(s) and not MOJI.search(s):
return False
try:
back = s.encode("latin-1", "ignore").decode("gbk", "strict")
except (UnicodeEncodeError, UnicodeDecodeError):
return False
return bool(re.search(r"[一-鿿]", back)) and not re.search(r"[一-鿿]", s)
REPL = re.compile(r"[\ufffd\uFFFE\uFFFF]")
INVIS = re.compile(r"[​-‏

­᠎]")
CTRL = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]")
PAREN_FULL = re.compile(r"[()]")
PAREN_HALF = re.compile(r"\(|\)")
WS_RUN = re.compile(r"[ \u3000]{3,}")
# 括号全半角只查关联键列,长正文里两种都有属原文形态,不是缺陷
KEYCOL = re.compile(r"^院校$|^学院$|名称$|^school$|^dwmc$|^yxsmc$|^zymc$|^专业")
TABLES = ("src_target_major_checked", "src_master", "notices_all", "chsi_dir", "chsi_align",
"att_fields", "errata", "fee_check", "sources")
def main():
con = sqlite3.connect("file:%s?mode=ro" % DB.replace("\\", "/"), uri=True)
con.row_factory = sqlite3.Row
hits = Counter()
where = defaultdict(list)
mixed_paren = defaultdict(set)
for tb in TABLES:
try:
cols = [r[1] for r in con.execute("PRAGMA table_info(%s)" % tb) if r[2].upper() != "INTEGER"]
except sqlite3.Error:
continue
if not cols:
continue
idcol = None
for cand in ("zyyjfxdm", "notice_key", "院校", "school", "rowid"):
if cand in cols or cand == "rowid":
idcol = cand
break
for col in cols:
q = 'SELECT "%s" v, "%s" k FROM %s WHERE "%s" IS NOT NULL' % (col, idcol or col, tb, col) \
if idcol else 'SELECT "%s" v FROM %s WHERE "%s" IS NOT NULL' % (col, tb, col)
try:
cur = con.execute(q)
except sqlite3.Error:
continue
for r in cur:
s = r["v"]
if not isinstance(s, str):
continue
key = "%s.%s" % (tb, col)
if is_mojibake(s):
hits["mojibake残留"] += 1
if len(where["mojibake残留"]) < 6:
where["mojibake残留"].append("%s | %s | %s" % (key, str(r["k"])[:26], s[:52]))
for pat, name in ((REPL, "替换符U+FFFD"), (INVIS, "零宽/不可见字符"),
(CTRL, "控制字符"), (WS_RUN, "连续空白")):
m = pat.search(s)
if m:
hits[name] += 1
if len(where[name]) < 6:
where[name].append("%s | %s | %s" % (
key, str(r["k"])[:26] if "k" in r.keys() else "",
s[max(0, m.start() - 12):m.start() + 26]))
if KEYCOL.search(col) and PAREN_FULL.search(s) and PAREN_HALF.search(s):
hits["键列括号全半角混用"] += 1
mixed_paren[key].add(s[:40])
con.close()
tot = sum(hits.values())
print("编码体检命中合计 %d 处:" % tot)
for k, v in hits.most_common():
print(" %-22s %5d" % (k, v))
# 归属分层:src_* 是源 CSV 镜像、core_* 是站点分片镜像,两者都按约定不动,
# 脏字符在读取侧 NFKC 归一解决(关联与导出都已实现)。只有自建表命中才算缺陷。
con2 = sqlite3.connect("file:%s?mode=ro" % DB.replace("\\", "/"), uri=True)
own = Counter()
for (t,) in con2.execute("SELECT name FROM sqlite_master WHERE type='table'"):
if t.startswith("src_") or t.startswith("core_"):
continue
try:
cols = [r[1] for r in con2.execute("PRAGMA table_info(%s)" % t) if r[2].upper() != "INTEGER"]
except sqlite3.Error:
continue
for col in cols:
try:
cur = con2.execute('SELECT "%s" v FROM %s WHERE "%s" IS NOT NULL' % (col, t, col))
except sqlite3.Error:
continue
for (v,) in cur:
if isinstance(v, str):
for pat, name in ((CTRL, "控制字符"), (INVIS, "零宽"), (REPL, "替换符")):
if pat.search(v):
own["%s.%s:%s" % (t, col, name)] += 1
con2.close()
n_own = sum(own.values())
print("\n归属分层:")
print(" 自建表(我方负责,须为 0):%d 处 %s" % (n_own, dict(own) if own else ""))
print(" 其余 %d 处落在 src_*/core_* 镜像与公告正文:按约定与源保持一致,不改值,读取侧归一"
% (tot - n_own))
for k, ex in where.items():
print("\n[%s] 样例" % k)
for e in ex[:4]:
print(" " + e.replace("\n", "⏎")[:130])
if mixed_paren:
print("\n括号全半角混用的列:")
for k, v in list(mixed_paren.items())[:8]:
print(" %-40s %d 种,例:%s" % (k, len(v), list(v)[0]))
print("\n结论:括号形态按来源不同,关联前必须 NFKC 归一(align_chsi.nm 已实现);"
"其余命中逐条判是否入库前就要清。")
return 0 if tot == 0 else 2
if __name__ == "__main__":
sys.exit(main())