Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 63 additions & 0 deletions .github/workflows/test-text-postprocessing.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
name: Validate text postprocessing

on:
pull_request:
paths:
- "funasr/utils/postprocess_hotwords.py"
- "funasr/utils/postprocess_utils.py"
- "runtime/python/onnxruntime/funasr_onnx/utils/postprocess_utils.py"
- "runtime/python/libtorch/funasr_torch/utils/postprocess_utils.py"
- "tests/test_postprocess_hotwords.py"
- "tests/test_rich_transcription_postprocess.py"
- ".github/workflows/test-text-postprocessing.yml"
push:
branches: [main]
paths:
- "funasr/utils/postprocess_hotwords.py"
- "funasr/utils/postprocess_utils.py"
- "runtime/python/onnxruntime/funasr_onnx/utils/postprocess_utils.py"
- "runtime/python/libtorch/funasr_torch/utils/postprocess_utils.py"
- "tests/test_postprocess_hotwords.py"
- "tests/test_rich_transcription_postprocess.py"
- ".github/workflows/test-text-postprocessing.yml"

permissions:
contents: read

jobs:
text-postprocessing:
runs-on: ubuntu-24.04
timeout-minutes: 10
strategy:
fail-fast: false
matrix:
python: ["3.10", "3.12"]
steps:
- uses: actions/checkout@v4
with:
ref: ${{ github.event.pull_request.head.sha || github.sha }}
- uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python }}
- name: Install text test dependencies
run: python -m pip install pytest==8.1.1 pypinyin==0.55.0 rapidfuzz==3.14.5
- name: Record source and dependencies
run: |
git rev-parse HEAD | tee source-commit.txt
python -m pip freeze | tee resolved-dependencies.txt
python -m pip check
python -c "import pypinyin, rapidfuzz"
- name: Check hotword and rich transcription behavior without model weights
run: |
python -m pytest -q \
tests/test_postprocess_hotwords.py \
tests/test_rich_transcription_postprocess.py \
--junitxml=text-postprocessing-results.xml
- uses: actions/upload-artifact@v4
if: always()
with:
name: text-postprocessing-python-${{ matrix.python }}
path: |
source-commit.txt
resolved-dependencies.txt
text-postprocessing-results.xml
5 changes: 5 additions & 0 deletions funasr/utils/postprocess_hotwords.py
Original file line number Diff line number Diff line change
Expand Up @@ -364,6 +364,11 @@ def _apply_fuzzy(self, text: str) -> Tuple[str, List[HotwordMatch]]:
# replace a Chinese character read "ai".
if (segment_latin and not target_latin) or (segment_han and not target_han):
continue
# Cross-script correction needs a complete homophone:
# "ai zhushou" can become "AI助手", but "duan" must
# not gain the missing Latin prefix in "C端".
if target_latin and not segment_latin and segment_py != target_py:
continue
score = self._fuzz.ratio(segment_py, target_py) / 100.0
if score >= self.threshold:
candidates.append(
Expand Down
17 changes: 17 additions & 0 deletions tests/test_postprocess_hotwords.py
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,23 @@ def test_fuzzy_keeps_latin_and_chinese_apart(self):
matcher = PostprocessHotwordMatcher(fuzzy_targets=["AI助手"])
self.assertEqual(matcher.apply_text("打开爱助手")[0], "打开AI助手")

def test_fuzzy_mixed_target_requires_complete_cross_script_homophone(self):
try:
import pypinyin # noqa: F401
import rapidfuzz # noqa: F401
except ImportError:
self.skipTest("pypinyin and rapidfuzz are required for fuzzy tests")

matcher = PostprocessHotwordMatcher(fuzzy_targets=["C端"])
for text in ["这个端口", "端", "端口2024", "C端产品"]:
with self.subTest(text=text):
self.assertEqual(matcher.apply_text(text), (text, []))
self.assertEqual(matcher.apply_text("C段产品")[0], "C端产品")

matcher = PostprocessHotwordMatcher(fuzzy_targets=["AI助手"])
self.assertEqual(matcher.apply_text("打开爱助手")[0], "打开AI助手")
self.assertEqual(matcher.apply_text("打开AI助守")[0], "打开AI助手")

def test_missing_fuzzy_dependency_raises(self):
with mock.patch.object(
postprocess_hotwords,
Expand Down
Loading