Files
speaking-practice/backend/subtitle_alignment.py
T

125 lines
4.3 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
from __future__ import annotations
from dataclasses import dataclass
from pathlib import Path
import subprocess
import tempfile
from typing import Callable
from .subtitle_tools import SubtitleCue, parse_timestamp
FIRST_SENTENCE_ALIGNMENT_PROMPT = """
你是字幕首句时间校准助手。你只需要分析这段音频的前 10 秒。
请找出给定英文字幕第一句在音频中真正开始说话的时间,单位是秒。
只返回 JSON:{"first_sentence_start":1.23,"matched_text":"...","confidence":0.95}
不要返回解释,不要校准后续句子。
""".strip()
@dataclass
class AlignmentResult:
applied: bool
offset_seconds: float
first_sentence_start: float | None
confidence: float
note: str
def align_first_sentence(video_path: Path, first_cue: SubtitleCue, client) -> AlignmentResult:
if not video_path.exists():
return AlignmentResult(False, 0.0, None, 0.0, "视频文件不存在,已跳过自动首句校准。")
with tempfile.TemporaryDirectory() as temp_dir:
audio_path = Path(temp_dir) / "first10.wav"
try:
extract_first_audio(video_path, audio_path)
response = client.audio_json(
FIRST_SENTENCE_ALIGNMENT_PROMPT,
audio_path,
f"第一句英文字幕:{first_cue.text}\n请只判断这句开始说话的秒数。",
)
except Exception as error:
return AlignmentResult(False, 0.0, None, 0.0, f"首句自动校准失败,已跳过:{error}")
return alignment_from_start(first_cue, response.get("first_sentence_start"), response.get("confidence"))
def align_first_sentence_with_asr(
video_path: Path,
first_cue: SubtitleCue,
asr_client,
extract_audio: Callable[[Path, Path], None] = None,
) -> AlignmentResult:
if not video_path.exists():
return AlignmentResult(False, 0.0, None, 0.0, "视频文件不存在,已跳过 OpenAI ASR 首句校准。")
audio_extractor = extract_audio or extract_first_audio
with tempfile.TemporaryDirectory() as temp_dir:
audio_path = Path(temp_dir) / "first10.wav"
try:
audio_extractor(video_path, audio_path)
response = asr_client.transcribe(audio_path)
except Exception as error:
return AlignmentResult(False, 0.0, None, 0.0, f"OpenAI ASR 首句校准失败,已跳过:{error}")
first_start = first_segment_start(response)
return alignment_from_start(first_cue, first_start, 0.9, source_name="OpenAI ASR")
def alignment_from_start(
first_cue: SubtitleCue,
start_value,
confidence_value=None,
source_name: str = "AI",
) -> AlignmentResult:
first_sentence_start = parse_alignment_second(start_value)
confidence = parse_alignment_second(confidence_value)
if first_sentence_start is None:
return AlignmentResult(False, 0.0, None, confidence or 0.0, f"{source_name} 没有返回有效的首句开始时间,已跳过自动校准。")
offset = first_sentence_start - parse_timestamp(first_cue.start)
if abs(offset) > 5:
return AlignmentResult(False, 0.0, first_sentence_start, confidence or 0.0, f"检测到偏移 {offset:.2f} 秒,超过 5 秒安全阈值,已跳过自动校准。")
return AlignmentResult(True, offset, first_sentence_start, confidence or 0.0, f"{source_name} 首句校准成功,整体平移 {offset:.2f} 秒。")
def first_segment_start(response: dict[str, object]) -> float | None:
segments = response.get("segments")
if isinstance(segments, list):
for segment in segments:
if isinstance(segment, dict):
start = parse_alignment_second(segment.get("start"))
text = str(segment.get("text", "")).strip()
if start is not None and text:
return start
return None
def extract_first_audio(video_path: Path, audio_path: Path) -> None:
command = [
"ffmpeg",
"-y",
"-i",
str(video_path),
"-ss",
"0",
"-t",
"10",
"-vn",
"-ac",
"1",
"-ar",
"16000",
str(audio_path),
]
subprocess.run(command, check=True, capture_output=True)
def parse_alignment_second(value) -> float | None:
try:
return float(value)
except (TypeError, ValueError):
return None