From 2f8daeb83bf15a7436f78eff75b9978d5633b657 Mon Sep 17 00:00:00 2001 From: LLM prompted by Jesse Bickel Date: Tue, 10 Nov 2026 11:02:10 -0600 Subject: [PATCH] Add ruff and bandit Python checks to CI The standalone Python scripts ship with no static analysis: CI only lints the TypeScript side and prettier ignores the standalone directory. Gate ruff (lint plus format check) and bandit over the scripts' explicit paths, and fix what they surface, including a Keycloak API request made without a timeout. GLM-5.3-Flash, an LLM supervised and prompted by @bickelj --- .github/workflows/ci.yml | 26 ++++++++++++++++++++++++++ standalone/pdc-optimize-field-matching | 8 ++++---- standalone/pdc-process-base-fields | 21 +++++++++++++-------- standalone/ruff.toml | 22 ++++++++++++++++++++++ 4 files changed, 65 insertions(+), 12 deletions(-) create mode 100644 standalone/ruff.toml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b80ddec..9a73be1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -64,6 +64,32 @@ jobs: - run: npm ci - run: npm run lint:tsc + # The standalone scripts carry no .py extension on purpose, so ruff + # and bandit lint their explicit paths below; a directory scan would + # silently skip them. See standalone/ruff.toml for the rule set. + ruff: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: astral-sh/ruff-action@v4 + with: + version: "0.17.4" + args: check standalone/pdc-process-base-fields standalone/pdc-optimize-field-matching + - uses: astral-sh/ruff-action@v4 + with: + version: "0.17.4" + args: format --check standalone/pdc-process-base-fields standalone/pdc-optimize-field-matching + + bandit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 + with: + python-version: "3.13" + - run: pip install bandit==1.9.4 + - run: bandit -q standalone/pdc-process-base-fields standalone/pdc-optimize-field-matching + build: runs-on: ubuntu-latest steps: diff --git a/standalone/pdc-optimize-field-matching b/standalone/pdc-optimize-field-matching index 4737947..cc204e1 100755 --- a/standalone/pdc-optimize-field-matching +++ b/standalone/pdc-optimize-field-matching @@ -138,6 +138,7 @@ def retrieve_pdc_data(endpoint: str, *, params: dict[str, Any] | None = None): f"{DS_PDC_API_BASE_URL}{endpoint}", headers={"Authorization": f"Bearer {access_token}"}, params=params or {}, + timeout=60, ) pdc_data_dict = pdc_data_response.json() @@ -231,7 +232,6 @@ class RAG(dspy.Module): ) - @app.command() def retrieve_application_forms(): retrieve_pdc_data( @@ -250,7 +250,7 @@ def retrieve_base_fields(): @app.command() def transform_application_forms(): - with open(base_path / "applicationForms.json", "r", encoding="utf-8") as f: + with open(base_path / "applicationForms.json", encoding="utf-8") as f: application_form_dict = json.load(f) mappings = [] @@ -396,7 +396,7 @@ def optimize_rag(): num_threads=2, track_stats=True, use_merge=False, - reflection_lm=lm, # TODO: use a separate reflection LLM + reflection_lm=lm, # TODO: use a separate reflection LLM ) rag = RAG() @@ -414,7 +414,7 @@ def optimize_rag(): optimized_rag = optimizer.compile( rag, trainset=trainset_examples, - valset=trainset_examples, # TODO: use a separate validation set + valset=trainset_examples, # TODO: use a separate validation set ) optimized_rag.save(base_path / "optimized_rag.json") diff --git a/standalone/pdc-process-base-fields b/standalone/pdc-process-base-fields index 8486699..a9e6d5b 100755 --- a/standalone/pdc-process-base-fields +++ b/standalone/pdc-process-base-fields @@ -69,8 +69,8 @@ those columns can simply do so. } """ -import json import argparse +import json import sys try: @@ -107,14 +107,18 @@ def main(): # here would be RawDescriptionHelpFormatter -- I don't usually # use that because it can do some minor reformatting that I # find aesthetically displeasing, but hey, YMMV. - formatter_class=argparse.RawTextHelpFormatter) - allowed_formats = ["csv",] # maybe also "markdown", "html", etc some day + formatter_class=argparse.RawTextHelpFormatter, + ) + allowed_formats = [ + "csv", + ] # maybe also "markdown", "html", etc some day arg_parser.add_argument( - '--output-format', + "--output-format", choices=allowed_formats, type=str.lower, default="csv", - help="Output format to show base fields in. Default: 'csv'") + help="Output format to show base fields in. Default: 'csv'", + ) # Parse command line. args = arg_parser.parse_args() @@ -123,11 +127,12 @@ def main(): json_input = json.load(sys.stdin) parsed_input = pandas.json_normalize(json_input) # Switch to index=True to include a first column that shows DataFrame index - parsed_input.to_csv(sys.stdout, index=False, encoding='utf-8') + parsed_input.to_csv(sys.stdout, index=False, encoding="utf-8") else: # can't happen raise ValueError( - f"ERROR: Unknown output format 'f{args.output_format}' requested") + f"ERROR: Unknown output format 'f{args.output_format}' requested" + ) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/standalone/ruff.toml b/standalone/ruff.toml new file mode 100644 index 0000000..9a7b813 --- /dev/null +++ b/standalone/ruff.toml @@ -0,0 +1,22 @@ +# Ruff configuration for the standalone Python scripts. +# +# The scripts deliberately carry no .py extension (they are standalone +# command-line programs), so CI passes their paths to ruff and bandit +# explicitly; a directory scan would silently skip them. +target-version = "py311" + +[lint] +select = [ + "B", # flake8-bugbear + "C4", # flake8-comprehensions + "E4", # pycodestyle imports + "E7", # pycodestyle statements + "E9", # pycodestyle runtime + "F", # pyflakes + "I", # isort + "SIM", # flake8-simplify + "UP", # pyupgrade + "W", # pycodestyle whitespace +] +# E501 (line-too-long) is not selected: the scripts embed long example +# command lines and URLs in their docstrings on purpose.