diff --git a/.github/workflows/test-text-postprocessing.yml b/.github/workflows/test-text-postprocessing.yml new file mode 100644 index 0000000000..e76358e8ee --- /dev/null +++ b/.github/workflows/test-text-postprocessing.yml @@ -0,0 +1,63 @@ +name: Validate text postprocessing + +on: + pull_request: + paths: + - "funasr/utils/postprocess_hotwords.py" + - "funasr/utils/postprocess_utils.py" + - "runtime/python/onnxruntime/funasr_onnx/utils/postprocess_utils.py" + - "runtime/python/libtorch/funasr_torch/utils/postprocess_utils.py" + - "tests/test_postprocess_hotwords.py" + - "tests/test_rich_transcription_postprocess.py" + - ".github/workflows/test-text-postprocessing.yml" + push: + branches: [main] + paths: + - "funasr/utils/postprocess_hotwords.py" + - "funasr/utils/postprocess_utils.py" + - "runtime/python/onnxruntime/funasr_onnx/utils/postprocess_utils.py" + - "runtime/python/libtorch/funasr_torch/utils/postprocess_utils.py" + - "tests/test_postprocess_hotwords.py" + - "tests/test_rich_transcription_postprocess.py" + - ".github/workflows/test-text-postprocessing.yml" + +permissions: + contents: read + +jobs: + text-postprocessing: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + python: ["3.10", "3.12"] + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + - name: Install text test dependencies + run: python -m pip install pytest==8.1.1 pypinyin==0.55.0 rapidfuzz==3.14.5 + - name: Record source and dependencies + run: | + git rev-parse HEAD | tee source-commit.txt + python -m pip freeze | tee resolved-dependencies.txt + python -m pip check + python -c "import pypinyin, rapidfuzz" + - name: Check hotword and rich transcription behavior without model weights + run: | + python -m pytest -q \ + tests/test_postprocess_hotwords.py \ + tests/test_rich_transcription_postprocess.py \ + --junitxml=text-postprocessing-results.xml + - uses: actions/upload-artifact@v4 + if: always() + with: + name: text-postprocessing-python-${{ matrix.python }} + path: | + source-commit.txt + resolved-dependencies.txt + text-postprocessing-results.xml diff --git a/funasr/utils/postprocess_hotwords.py b/funasr/utils/postprocess_hotwords.py index 5ff6183fa8..a86a2013ec 100644 --- a/funasr/utils/postprocess_hotwords.py +++ b/funasr/utils/postprocess_hotwords.py @@ -364,6 +364,11 @@ def _apply_fuzzy(self, text: str) -> Tuple[str, List[HotwordMatch]]: # replace a Chinese character read "ai". if (segment_latin and not target_latin) or (segment_han and not target_han): continue + # Cross-script correction needs a complete homophone: + # "ai zhushou" can become "AI助手", but "duan" must + # not gain the missing Latin prefix in "C端". + if target_latin and not segment_latin and segment_py != target_py: + continue score = self._fuzz.ratio(segment_py, target_py) / 100.0 if score >= self.threshold: candidates.append( diff --git a/tests/test_postprocess_hotwords.py b/tests/test_postprocess_hotwords.py index 330219d5ef..6dea9e2180 100644 --- a/tests/test_postprocess_hotwords.py +++ b/tests/test_postprocess_hotwords.py @@ -138,6 +138,23 @@ def test_fuzzy_keeps_latin_and_chinese_apart(self): matcher = PostprocessHotwordMatcher(fuzzy_targets=["AI助手"]) self.assertEqual(matcher.apply_text("打开爱助手")[0], "打开AI助手") + def test_fuzzy_mixed_target_requires_complete_cross_script_homophone(self): + try: + import pypinyin # noqa: F401 + import rapidfuzz # noqa: F401 + except ImportError: + self.skipTest("pypinyin and rapidfuzz are required for fuzzy tests") + + matcher = PostprocessHotwordMatcher(fuzzy_targets=["C端"]) + for text in ["这个端口", "端", "端口2024", "C端产品"]: + with self.subTest(text=text): + self.assertEqual(matcher.apply_text(text), (text, [])) + self.assertEqual(matcher.apply_text("C段产品")[0], "C端产品") + + matcher = PostprocessHotwordMatcher(fuzzy_targets=["AI助手"]) + self.assertEqual(matcher.apply_text("打开爱助手")[0], "打开AI助手") + self.assertEqual(matcher.apply_text("打开AI助守")[0], "打开AI助手") + def test_missing_fuzzy_dependency_raises(self): with mock.patch.object( postprocess_hotwords,