143 lines
6.2 KiB
Python
143 lines
6.2 KiB
Python
# -*- coding: utf-8 -*-
|
||
"""全库文本编码体检:找 mojibake、替换符、不可见字符、控制字符、括号全半角混用。
|
||
|
||
动机:对齐时发现校名「中国石油大学(北京)」官方半角括号、平台全角括号,
|
||
52 行因此没匹配上。同类问题会静默压低任何按名称关联的命中率,所以要一次扫干净。
|
||
只报告不改数据;要改的进 errata。
|
||
"""
|
||
import os
|
||
import re
|
||
import sqlite3
|
||
import sys
|
||
import unicodedata
|
||
from collections import Counter, defaultdict
|
||
|
||
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
|
||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||
DB = os.path.join(HERE, "zexiao_2027.db")
|
||
|
||
# mojibake 判据收紧:单看字符形状会误伤(「〔2013〕19号」是公文正常括号,不是乱码)。
|
||
# 真乱码的特征是「按 latin-1 重编回去、再用 gbk 能解出中文」,且串里拉丁扩展字符成串出现。
|
||
LATIN_RUN = re.compile(r"[À-ÿ]{2,}")
|
||
MOJI = re.compile(r"(锟斤拷|拷拷|â€|Ã.)")
|
||
|
||
|
||
def is_mojibake(s):
|
||
if not LATIN_RUN.search(s) and not MOJI.search(s):
|
||
return False
|
||
try:
|
||
back = s.encode("latin-1", "ignore").decode("gbk", "strict")
|
||
except (UnicodeEncodeError, UnicodeDecodeError):
|
||
return False
|
||
return bool(re.search(r"[一-鿿]", back)) and not re.search(r"[一-鿿]", s)
|
||
|
||
|
||
REPL = re.compile(r"[\ufffd\uFFFE\uFFFF]")
|
||
INVIS = re.compile(r"[-
]")
|
||
CTRL = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]")
|
||
PAREN_FULL = re.compile(r"[()]")
|
||
PAREN_HALF = re.compile(r"\(|\)")
|
||
WS_RUN = re.compile(r"[ \u3000]{3,}")
|
||
# 括号全半角只查关联键列,长正文里两种都有属原文形态,不是缺陷
|
||
KEYCOL = re.compile(r"^院校$|^学院$|名称$|^school$|^dwmc$|^yxsmc$|^zymc$|^专业")
|
||
TABLES = ("src_target_major_checked", "src_master", "notices_all", "chsi_dir", "chsi_align",
|
||
"att_fields", "errata", "fee_check", "sources")
|
||
|
||
|
||
def main():
|
||
con = sqlite3.connect("file:%s?mode=ro" % DB.replace("\\", "/"), uri=True)
|
||
con.row_factory = sqlite3.Row
|
||
hits = Counter()
|
||
where = defaultdict(list)
|
||
mixed_paren = defaultdict(set)
|
||
|
||
for tb in TABLES:
|
||
try:
|
||
cols = [r[1] for r in con.execute("PRAGMA table_info(%s)" % tb) if r[2].upper() != "INTEGER"]
|
||
except sqlite3.Error:
|
||
continue
|
||
if not cols:
|
||
continue
|
||
idcol = None
|
||
for cand in ("zyyjfxdm", "notice_key", "院校", "school", "rowid"):
|
||
if cand in cols or cand == "rowid":
|
||
idcol = cand
|
||
break
|
||
for col in cols:
|
||
q = 'SELECT "%s" v, "%s" k FROM %s WHERE "%s" IS NOT NULL' % (col, idcol or col, tb, col) \
|
||
if idcol else 'SELECT "%s" v FROM %s WHERE "%s" IS NOT NULL' % (col, tb, col)
|
||
try:
|
||
cur = con.execute(q)
|
||
except sqlite3.Error:
|
||
continue
|
||
for r in cur:
|
||
s = r["v"]
|
||
if not isinstance(s, str):
|
||
continue
|
||
key = "%s.%s" % (tb, col)
|
||
if is_mojibake(s):
|
||
hits["mojibake残留"] += 1
|
||
if len(where["mojibake残留"]) < 6:
|
||
where["mojibake残留"].append("%s | %s | %s" % (key, str(r["k"])[:26], s[:52]))
|
||
for pat, name in ((REPL, "替换符U+FFFD"), (INVIS, "零宽/不可见字符"),
|
||
(CTRL, "控制字符"), (WS_RUN, "连续空白")):
|
||
m = pat.search(s)
|
||
if m:
|
||
hits[name] += 1
|
||
if len(where[name]) < 6:
|
||
where[name].append("%s | %s | %s" % (
|
||
key, str(r["k"])[:26] if "k" in r.keys() else "",
|
||
s[max(0, m.start() - 12):m.start() + 26]))
|
||
if KEYCOL.search(col) and PAREN_FULL.search(s) and PAREN_HALF.search(s):
|
||
hits["键列括号全半角混用"] += 1
|
||
mixed_paren[key].add(s[:40])
|
||
con.close()
|
||
|
||
tot = sum(hits.values())
|
||
print("编码体检命中合计 %d 处:" % tot)
|
||
for k, v in hits.most_common():
|
||
print(" %-22s %5d" % (k, v))
|
||
|
||
# 归属分层:src_* 是源 CSV 镜像、core_* 是站点分片镜像,两者都按约定不动,
|
||
# 脏字符在读取侧 NFKC 归一解决(关联与导出都已实现)。只有自建表命中才算缺陷。
|
||
con2 = sqlite3.connect("file:%s?mode=ro" % DB.replace("\\", "/"), uri=True)
|
||
own = Counter()
|
||
for (t,) in con2.execute("SELECT name FROM sqlite_master WHERE type='table'"):
|
||
if t.startswith("src_") or t.startswith("core_"):
|
||
continue
|
||
try:
|
||
cols = [r[1] for r in con2.execute("PRAGMA table_info(%s)" % t) if r[2].upper() != "INTEGER"]
|
||
except sqlite3.Error:
|
||
continue
|
||
for col in cols:
|
||
try:
|
||
cur = con2.execute('SELECT "%s" v FROM %s WHERE "%s" IS NOT NULL' % (col, t, col))
|
||
except sqlite3.Error:
|
||
continue
|
||
for (v,) in cur:
|
||
if isinstance(v, str):
|
||
for pat, name in ((CTRL, "控制字符"), (INVIS, "零宽"), (REPL, "替换符")):
|
||
if pat.search(v):
|
||
own["%s.%s:%s" % (t, col, name)] += 1
|
||
con2.close()
|
||
n_own = sum(own.values())
|
||
print("\n归属分层:")
|
||
print(" 自建表(我方负责,须为 0):%d 处 %s" % (n_own, dict(own) if own else ""))
|
||
print(" 其余 %d 处落在 src_*/core_* 镜像与公告正文:按约定与源保持一致,不改值,读取侧归一"
|
||
% (tot - n_own))
|
||
for k, ex in where.items():
|
||
print("\n[%s] 样例" % k)
|
||
for e in ex[:4]:
|
||
print(" " + e.replace("\n", "⏎")[:130])
|
||
if mixed_paren:
|
||
print("\n括号全半角混用的列:")
|
||
for k, v in list(mixed_paren.items())[:8]:
|
||
print(" %-40s %d 种,例:%s" % (k, len(v), list(v)[0]))
|
||
print("\n结论:括号形态按来源不同,关联前必须 NFKC 归一(align_chsi.nm 已实现);"
|
||
"其余命中逐条判是否入库前就要清。")
|
||
return 0 if tot == 0 else 2
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|