build: package dashboard with cythonized core

This commit is contained in:
Your Name committed 2026-06-23 23:30:29 +08:00
1 parent 4dea1328ec
commit 008d7f82b3
9 files changed
+418 -13

No files matched your search

@@ -6,6 +6,7 @@ import os
from typing import Any
from douyin.fudai.core.coords import (
DETAIL_OUTSIDE_CLICK,
FANSCLUB_JOIN_CLICK,
FANSCLUB_JOIN_FALLBACK_CLICK,
)
@@ -236,8 +236,6 @@ def validate_fudai_detail(detail: Tuple, analyser: Any = None) -> Tuple[bool, st
value_raw = str(getattr(analyser, "value_raw", "") or "").strip()
if value_raw and prize_value_yuan > 0:
# 检查原始文本是否包含合理的价值信息
import re
# 查找所有可能金额。营销文案常见“0元带走¥999参考价值”,
# 不能只看第一个数字,否则会把已正确解析出的参考价值误判失败。
value_patterns = [
+6 -6
View File
@@ -19,20 +19,20 @@ powershell -NoProfile -ExecutionPolicy Bypass -File packaging\pyinstaller\build_
powershell -NoProfile -ExecutionPolicy Bypass -File packaging\pyinstaller\build_windows.ps1 -SkipInstall
```
正式打包默认会先通过 PyArmor 生成混淆 staging,再让 PyInstaller 从混淆目录打包:
正式打包默认会先通过 Cython 编译 `douyin/` 和 `operations/` 核心模块,再让 PyInstaller 从 `build\cythonized-src\` 打包:
```powershell
powershell -NoProfile -ExecutionPolicy Bypass -File packaging\pyinstaller\build_windows.ps1 -SkipInstall
```
混淆流程会先生成 `build\obfuscated-src\`,再让 PyInstaller 从该目录打包。该目录是中间产物,不要直接分发;最终仍然只分发 `dist\DouyinGuaji-<version>\` 或 Inno 生成的安装包。
该流程会生成 `build\cythonized-src\`,其中核心模块以 `.pyd` 形式存在。该目录是中间产物,不要直接分发;最终仍然只分发 `dist\DouyinGuaji-<version>\` 或 Inno 生成的安装包。
注意:PyArmor 试用版会在混淆文件头部留下 trial 标记,正式分发前建议购买/配置正式授权后重新打包。若 PyArmor 授权不可用,正式打包会失败并停止。
如果需要继续使用旧的 PyArmor 路线,可额外加 `-UsePyArmor`。注意:PyArmor 试用版会在混淆文件头部留下 trial 标记,正式分发前建议购买/配置正式授权后重新打包。若 PyArmor 授权不可用,旧路线会失败并停止。
仅本机开发调试需要生成明文包时,才显式加 `-NoObfuscateForDevOnly`:
仅本机开发调试需要生成明文包时,才显式加 `-NoCythonForDevOnly`:
```powershell
powershell -NoProfile -ExecutionPolicy Bypass -File packaging\pyinstaller\build_windows.ps1 -SkipInstall -NoObfuscateForDevOnly
powershell -NoProfile -ExecutionPolicy Bypass -File packaging\pyinstaller\build_windows.ps1 -SkipInstall -NoCythonForDevOnly
```
如果要尝试对根目录 exe 做 UPX 加壳/压缩,可额外加 `-PackExe -UpxDir <UPX目录>`:
@@ -93,4 +93,4 @@ dist\installer\DouyinGuaji-3.3.3-Setup.exe
- 目标电脑需要能使用 ADB,或者把 Android platform-tools 加入 PATH。
- 首次启动 PaddleOCR 相关组件可能较慢,杀毒软件也可能扫描 exe。
- `config/config.json` 是本地运行配置,默认不随安装包带个人配置;让使用者首次打开后在 GUI 中保存配置。
- 默认混淆后仍要以 `python -m tools.distribution_manifest dist\DouyinGuaji-<version>` 的结果为准;PASS 才能发给别人。使用 `-NoObfuscateForDevOnly` 生成的明文包禁止分发。
- 默认 Cython 编译后仍要以 `python -m tools.distribution_manifest dist\DouyinGuaji-<version>` 的结果为准;PASS 才能发给别人。使用 `-NoCythonForDevOnly` 生成的明文包禁止分发。
+13 -3
View File
@@ -1,6 +1,8 @@
param(
[string]$Version = "",
[switch]$SkipInstall,
[switch]$NoCythonForDevOnly,
[switch]$UsePyArmor,
[switch]$NoObfuscateForDevOnly,
[switch]$PackExe,
[switch]$PackExeForce,
@@ -27,8 +29,10 @@ if (-not $SkipInstall) {
python -m pip install --upgrade pip
python -m pip install -r requirements.txt
python -m pip install pyinstaller
if (-not $NoObfuscateForDevOnly) {
if ($UsePyArmor) {
python -m pip install pyarmor
} elseif (-not $NoCythonForDevOnly -and -not $NoObfuscateForDevOnly) {
python -m pip install cython setuptools wheel
}
}
@@ -40,12 +44,18 @@ New-Item -ItemType Directory -Force $DistRoot | Out-Null
$BuildSourceRoot = $Root
$HiddenImportsFile = Join-Path $Root "build\obfuscation-hidden-imports.json"
if (-not $NoObfuscateForDevOnly) {
if ($UsePyArmor -and -not $NoObfuscateForDevOnly) {
$BuildSourceRoot = Join-Path $Root "build\obfuscated-src"
python -m tools.obfuscation_staging --source-root $Root --staging-root $BuildSourceRoot --hidden-imports-output $HiddenImportsFile
if ($LASTEXITCODE -ne 0) {
throw "PyArmor obfuscation failed with exit code $LASTEXITCODE"
}
} elseif (-not $NoCythonForDevOnly -and -not $NoObfuscateForDevOnly) {
$BuildSourceRoot = Join-Path $Root "build\cythonized-src"
python -m tools.cython_staging --source-root $Root --staging-root $BuildSourceRoot --hidden-imports-output $HiddenImportsFile
if ($LASTEXITCODE -ne 0) {
throw "Cython core compile failed with exit code $LASTEXITCODE"
}
} else {
Write-Warning "Development plaintext build requested. Do not distribute this package."
}
@@ -88,7 +98,7 @@ foreach ($name in $ExcludedOptionalModules) {
$CommonArgs += @("--exclude-module", $name)
}
if ((-not $NoObfuscateForDevOnly) -and (Test-Path $HiddenImportsFile)) {
if ((-not $NoObfuscateForDevOnly) -and (-not $NoCythonForDevOnly -or $UsePyArmor) -and (Test-Path $HiddenImportsFile)) {
$HiddenImports = Get-Content -LiteralPath $HiddenImportsFile -Raw | ConvertFrom-Json
$CliOnlyHiddenImportRoots = @("cv2", "numpy", "paddleocr", "paddle")
foreach ($name in $HiddenImports) {
+101
View File
@@ -0,0 +1,101 @@
from __future__ import annotations
import subprocess
from pathlib import Path
import pytest
def _write(path: Path, text: str = "x = 1\n") -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(text, encoding="utf-8")
def _make_minimal_project(root: Path) -> None:
_write(root / "douyin" / "__init__.py")
_write(root / "douyin" / "fudai" / "__init__.py")
_write(root / "douyin" / "fudai" / "logic.py", "def answer():\n return 42\n")
_write(root / "douyin" / "core" / "__init__.py")
_write(root / "douyin" / "core" / "support.py", "def support():\n return 'plain support'\n")
_write(root / "operations" / "__init__.py")
_write(root / "operations" / "ocr_operations.py", "def read():\n return 'ok'\n")
_write(root / "gui" / "gui_dashboard_app_entry.py")
_write(root / "gui" / "web_gui_entry.py")
_write(root / "bin" / "run_cli.py")
_write(root / "bin" / "run_cli_entry.py")
_write(root / "tools" / "license_dev.py", "PRIVATE_KEY = 'must-not-ship'\n")
def test_create_cython_staging_compiles_core_modules(tmp_path, monkeypatch):
from tools import cython_staging
root = tmp_path / "project"
staging = tmp_path / "stage"
_make_minimal_project(root)
calls: list[list[str]] = []
def fake_runner(cmd, *, cwd, check, env=None):
calls.append([str(x) for x in cmd])
assert cwd == staging
assert check is True
assert env is not None
_write(staging / "douyin" / "fudai" / "logic.cp312-win_amd64.pyd", "binary")
return subprocess.CompletedProcess(cmd, 0)
result = cython_staging.create_cython_staging(root, staging, runner=fake_runner)
assert result.staging_root == staging
assert result.compiled_count == 1
assert calls[0][1] == "_cython_build_setup.py"
assert "build_ext" in calls[0]
assert not (staging / "douyin" / "fudai" / "logic.py").exists()
assert (staging / "douyin" / "core" / "support.py").exists()
assert (staging / "operations" / "ocr_operations.py").exists()
assert (staging / "douyin" / "__init__.py").exists()
assert (staging / "gui" / "gui_dashboard_app_entry.py").exists()
assert not (staging / "tools" / "license_dev.py").exists()
def test_validate_cython_staging_rejects_plain_compiled_source(tmp_path):
from tools.cython_staging import validate_cython_staging
staging = tmp_path / "stage"
_write(staging / "douyin" / "__init__.py")
_write(staging / "douyin" / "fudai" / "__init__.py")
_write(staging / "douyin" / "fudai" / "logic.py", "def leaked():\n return 'plain'\n")
_write(staging / "douyin" / "fudai" / "logic.cp312-win_amd64.pyd", "binary")
_write(staging / "operations" / "__init__.py")
_write(staging / "gui" / "gui_dashboard_app_entry.py")
_write(staging / "bin" / "run_cli.py")
_write(staging / "bin" / "run_cli_entry.py")
with pytest.raises(RuntimeError, match="plain compiled runtime source"):
validate_cython_staging(staging)
def test_validate_cython_staging_rejects_forbidden_entries(tmp_path):
from tools.cython_staging import validate_cython_staging
staging = tmp_path / "stage"
_write(staging / "douyin" / "__init__.py")
_write(staging / "douyin" / "fudai" / "__init__.py")
_write(staging / "douyin" / "fudai" / "logic.cp312-win_amd64.pyd", "binary")
_write(staging / "operations" / "__init__.py")
_write(staging / "gui" / "gui_dashboard_app_entry.py")
_write(staging / "bin" / "run_cli.py")
_write(staging / "bin" / "run_cli_entry.py")
_write(staging / "tools" / "license_dev.py", "PRIVATE_KEY = 'leak'\n")
with pytest.raises(RuntimeError, match="forbidden"):
validate_cython_staging(staging)
def test_windows_packaging_script_uses_cython_by_default():
script = Path("packaging/pyinstaller/build_windows.ps1").read_text(encoding="utf-8")
assert "[switch]$NoCythonForDevOnly" in script
assert "[switch]$UsePyArmor" in script
assert "tools.cython_staging" in script
assert "build\\cythonized-src" in script
assert "Cython core compile failed" in script
assert "Development plaintext build requested" in script
+2
View File
@@ -23,6 +23,7 @@ def test_distribution_manifest_blocks_source_and_debug_artifacts():
"webui/js/README.md",
"DouyinGuajiGUI/_internal/base_library.zip",
"_internal/douyin/core/douyin_fudai.py",
"_internal/douyin/fudai/douyin_fudai_core.cp312-win_amd64.pyd",
"_internal/cv2/__init__.py",
]
@@ -41,6 +42,7 @@ def test_distribution_manifest_blocks_source_and_debug_artifacts():
assert "webui/js/README.md" in blocked
assert "DouyinGuajiGUI/_internal/base_library.zip" in blocked
assert "_internal/douyin/core/douyin_fudai.py" in blocked
assert "_internal/douyin/fudai/douyin_fudai_core.cp312-win_amd64.pyd" not in blocked
assert "_internal/cv2/__init__.py" not in blocked
+7 -2
View File
@@ -210,11 +210,15 @@ def test_collect_runtime_hidden_imports_skips_optional_heavy_modules(tmp_path):
assert "scipy" not in hidden_imports
def test_windows_packaging_script_obfuscates_by_default():
def test_windows_packaging_script_protects_core_modules_by_default():
script = Path("packaging/pyinstaller/build_windows.ps1").read_text(encoding="utf-8")
assert "[switch]$NoCythonForDevOnly" in script
assert "[switch]$UsePyArmor" in script
assert "[switch]$NoObfuscateForDevOnly" in script
assert "if (-not $NoObfuscateForDevOnly)" in script
assert "if ($UsePyArmor -and -not $NoObfuscateForDevOnly)" in script
assert "elseif (-not $NoCythonForDevOnly -and -not $NoObfuscateForDevOnly)" in script
assert "tools.cython_staging" in script
assert "tools.obfuscation_staging" in script
assert '$env:DISABLE_MODEL_SOURCE_CHECK = "True"' in script
assert '$env:PADDLE_PDX_DISABLE_MODEL_SOURCE_CHECK = "1"' in script
@@ -233,6 +237,7 @@ def test_windows_packaging_script_obfuscates_by_default():
assert "python -m PyInstaller @CommonArgs @GuiArgs --name DouyinGuajiGUI" in script
assert "python -m PyInstaller @CommonArgs @GuiArgs --name web_gui" in script
assert "python -m PyInstaller @CommonArgs @CliArgs --name run_cli" in script
assert "Cython core compile failed" in script
assert "PyArmor obfuscation failed" in script
assert "Development plaintext build requested" in script
assert "--paths\", $BuildSourceRoot" in script
+286
View File
@@ -0,0 +1,286 @@
from __future__ import annotations
import argparse
import json
import os
import shutil
import subprocess
import sys
from dataclasses import dataclass
from pathlib import Path
from typing import Callable, Iterable, Sequence
from tools.obfuscation_staging import collect_runtime_hidden_imports
DEFAULT_COMPILED_SOURCE_DIRS = (
"douyin/fudai",
"douyin/reservation",
"douyin/detection",
"douyin/stage",
)
RUNTIME_SOURCE_DIRS = ("douyin", "operations")
PLAIN_SOURCE_DIRS = ("gui",)
RUNTIME_ENTRY_FILES = (
"bin/run_cli.py",
"bin/run_cli_entry.py",
)
FORBIDDEN_STAGING_PREFIXES = ("tests/", "artifacts/", "docs/", "packaging/")
FORBIDDEN_STAGING_FILES = ("tools/license_dev.py",)
EXTENSION_SUFFIXES = (".pyd", ".so")
Runner = Callable[..., subprocess.CompletedProcess]
@dataclass(frozen=True)
class CythonStagingResult:
staging_root: Path
compiled_count: int
def _as_posix(path: str | Path) -> str:
return str(path).replace("\\", "/")
def _iter_python_files(root: Path, source_dirs: Iterable[str]) -> list[Path]:
files: list[Path] = []
for rel in source_dirs:
base = root / rel
if base.exists():
files.extend(path for path in base.rglob("*.py") if path.is_file())
return sorted(files)
def _copy_runtime_sources(source: Path, target: Path) -> None:
for rel in [*RUNTIME_SOURCE_DIRS, *PLAIN_SOURCE_DIRS]:
src = source / rel
if src.exists():
shutil.copytree(src, target / rel, ignore=shutil.ignore_patterns("__pycache__"))
for rel in RUNTIME_ENTRY_FILES:
src = source / rel
if src.exists():
dst = target / rel
dst.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(src, dst)
def _write_setup_script(
staging_root: Path,
source_files: Sequence[Path],
*,
include_package_data: bool = False,
) -> Path:
rel_files = [_as_posix(path.relative_to(staging_root)) for path in source_files]
setup_path = staging_root / "_cython_build_setup.py"
if os.name == "nt":
(staging_root / "setup.cfg").write_text(
"[build_ext]\ncompiler=mingw32\n",
encoding="utf-8",
)
setup_path.write_text(
"\n".join(
[
"from __future__ import annotations",
"",
"from Cython.Build import cythonize",
"from setuptools import Extension, setup",
"",
f"SOURCES = {rel_files!r}",
"extensions = [",
" Extension(",
" source[:-3].replace('/', '.').replace('\\\\', '.'),",
" [source],",
" )",
" for source in SOURCES",
"]",
"",
"setup(",
" name='douyin_guaji_cythonized_runtime',",
" ext_modules=cythonize(",
" extensions,",
" compiler_directives={'language_level': '3'},",
" build_dir='_cython_c_build',",
" annotate=False,",
" ),",
f" include_package_data={include_package_data!r},",
")",
"",
]
),
encoding="utf-8",
)
return setup_path
def _remove_compiled_plain_sources(staging_root: Path, source_files: Sequence[Path]) -> None:
for path in source_files:
if path.name == "__init__.py":
continue
try:
path.unlink()
except FileNotFoundError:
pass
for generated in staging_root.rglob("*.c"):
generated.unlink()
for rel in ("_cython_build_setup.py", "setup.cfg", "_cython_c_build", "build"):
path = staging_root / rel
if path.is_dir():
shutil.rmtree(path)
elif path.exists():
path.unlink()
def _compiled_extension_exists_for(source_file: Path) -> bool:
base = source_file.with_suffix("")
parent = source_file.parent
prefix = base.name
return any(
candidate.is_file()
and candidate.name.startswith(prefix)
and candidate.suffix.lower() in EXTENSION_SUFFIXES
for candidate in parent.iterdir()
)
def _find_nuitka_cached_gcc_bin() -> Path | None:
cache = Path.home() / "AppData" / "Local" / "Nuitka" / "Nuitka" / "Cache" / "downloads" / "gcc"
if not cache.exists():
return None
for gcc in sorted(cache.rglob("gcc.exe"), key=lambda item: item.stat().st_mtime, reverse=True):
if gcc.parent.name.lower() == "bin":
return gcc.parent
return None
def validate_cython_staging(
staging_root: str | Path,
*,
compiled_source_dirs: Sequence[str] = DEFAULT_COMPILED_SOURCE_DIRS,
) -> CythonStagingResult:
root = Path(staging_root)
missing: list[str] = []
for rel in [*RUNTIME_SOURCE_DIRS, *PLAIN_SOURCE_DIRS, *RUNTIME_ENTRY_FILES]:
if not (root / rel).exists():
missing.append(rel)
if missing:
raise RuntimeError(f"cython staging missing required entries: {', '.join(missing)}")
forbidden: list[str] = []
for path in root.rglob("*"):
if not path.is_file():
continue
rel = path.relative_to(root).as_posix()
lower = rel.lower()
if any(lower.startswith(prefix) for prefix in FORBIDDEN_STAGING_PREFIXES):
forbidden.append(rel)
if lower in {item.lower() for item in FORBIDDEN_STAGING_FILES}:
forbidden.append(rel)
if forbidden:
raise RuntimeError(f"cython staging contains forbidden entries: {', '.join(forbidden[:20])}")
plain_sources: list[str] = []
compiled_count = 0
for source in _iter_python_files(root, compiled_source_dirs):
if source.name == "__init__.py":
continue
if _compiled_extension_exists_for(source):
plain_sources.append(source.relative_to(root).as_posix())
else:
# Missing extension is also a leak-risk: PyInstaller would package .py.
plain_sources.append(source.relative_to(root).as_posix())
for ext in root.rglob("*"):
if ext.is_file() and ext.suffix.lower() in EXTENSION_SUFFIXES:
compiled_count += 1
if plain_sources:
raise RuntimeError(
"cython staging contains plain compiled runtime source: "
+ ", ".join(plain_sources[:20])
)
if compiled_count == 0:
raise RuntimeError("cython staging contains no compiled extension modules")
return CythonStagingResult(staging_root=root, compiled_count=compiled_count)
def create_cython_staging(
source_root: str | Path,
staging_root: str | Path,
*,
compiled_source_dirs: Sequence[str] = DEFAULT_COMPILED_SOURCE_DIRS,
runner: Runner = subprocess.run,
) -> CythonStagingResult:
source = Path(source_root).resolve()
target = Path(staging_root).resolve()
if target.exists():
shutil.rmtree(target)
target.parent.mkdir(parents=True, exist_ok=True)
target.mkdir(parents=True, exist_ok=True)
_copy_runtime_sources(source, target)
source_files = [
path
for path in _iter_python_files(target, compiled_source_dirs)
if path.name != "__init__.py"
]
if not source_files:
raise RuntimeError("no cython runtime source files found")
setup_path = _write_setup_script(target, source_files)
cmd = [sys.executable, str(setup_path.name), "build_ext", "--inplace"]
env = os.environ.copy()
if os.name == "nt":
env.setdefault("DISTUTILS_USE_SDK", "0")
gcc_bin = _find_nuitka_cached_gcc_bin()
if gcc_bin is not None:
env["PATH"] = str(gcc_bin) + os.pathsep + env.get("PATH", "")
env.setdefault("CC", "gcc")
runner(cmd, cwd=target, check=True, env=env)
_remove_compiled_plain_sources(target, source_files)
return validate_cython_staging(target, compiled_source_dirs=compiled_source_dirs)
def main(argv: Sequence[str] | None = None) -> int:
parser = argparse.ArgumentParser(description="生成 PyInstaller 使用的 Cython 编译核心模块 staging 目录")
parser.add_argument("--source-root", default=".", help="项目根目录")
parser.add_argument("--staging-root", required=True, help="输出 staging 目录")
parser.add_argument(
"--compile-dir",
action="append",
dest="compile_dirs",
help="要 Cython 编译的源码目录;可重复传入。默认编译福袋/预约/检测/阶段核心包",
)
parser.add_argument(
"--hidden-imports-output",
default="",
help="可选:把原始运行源码扫描到的外部 hidden-import 写入 JSON 文件",
)
args = parser.parse_args(argv)
try:
hidden_imports = collect_runtime_hidden_imports(args.source_root) if args.hidden_imports_output else []
result = create_cython_staging(
args.source_root,
args.staging_root,
compiled_source_dirs=tuple(args.compile_dirs or DEFAULT_COMPILED_SOURCE_DIRS),
)
if args.hidden_imports_output:
output = Path(args.hidden_imports_output)
output.parent.mkdir(parents=True, exist_ok=True)
output.write_text(
json.dumps(hidden_imports, ensure_ascii=False, indent=2),
encoding="utf-8",
)
except subprocess.CalledProcessError as exc:
print(f"[CYTHON] FAIL build_ext exited with code {exc.returncode}", file=sys.stderr)
return exc.returncode or 1
except Exception as exc:
print(f"[CYTHON] FAIL {exc}", file=sys.stderr)
return 1
suffix = f" hidden_imports={len(hidden_imports)}" if args.hidden_imports_output else ""
print(f"[CYTHON] PASS staging={result.staging_root} compiled={result.compiled_count}{suffix}")
return 0
if __name__ == "__main__":
raise SystemExit(main())
+2
View File
@@ -106,6 +106,8 @@ def is_forbidden_distribution_entry(entry: str, *, allow_license: bool = False)
if lower == "license.dat" and allow_license:
return False
if lower.startswith("_internal/"):
if lower.endswith((".pyd", ".dll")):
return False
project_bits = {
"_internal/douyin/",
"_internal/gui/",