Add ruff and bandit Python checks to CI

.github/workflows/ci.yml

@@ -64,6 +64,32 @@ jobs:
       - run: npm ci
       - run: npm run lint:tsc
 
+  # The standalone scripts carry no .py extension on purpose, so ruff
+  # and bandit lint their explicit paths below; a directory scan would
+  # silently skip them. See standalone/ruff.toml for the rule set.
+  ruff:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+      - uses: astral-sh/ruff-action@v4
+        with:
+          version: "0.17.4"
+          args: check standalone/pdc-process-base-fields standalone/pdc-optimize-field-matching
+      - uses: astral-sh/ruff-action@v4
+        with:
+          version: "0.17.4"
+          args: format --check standalone/pdc-process-base-fields standalone/pdc-optimize-field-matching
+
+  bandit:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+      - uses: actions/setup-python@v7
+        with:
+          python-version: "3.13"
+      - run: pip install bandit==1.9.4
+      - run: bandit -q standalone/pdc-process-base-fields standalone/pdc-optimize-field-matching
+
   build:
     runs-on: ubuntu-latest
     steps:

standalone/pdc-optimize-field-matching

@@ -138,6 +138,7 @@ def retrieve_pdc_data(endpoint: str, *, params: dict[str, Any] | None = None):
         f"{DS_PDC_API_BASE_URL}{endpoint}",
         headers={"Authorization": f"Bearer {access_token}"},
         params=params or {},
+        timeout=60,
     )
 
     pdc_data_dict = pdc_data_response.json()
@@ -231,7 +232,6 @@ class RAG(dspy.Module):
         )
 
 
-
 @app.command()
 def retrieve_application_forms():
     retrieve_pdc_data(
@@ -250,7 +250,7 @@ def retrieve_base_fields():
 
 @app.command()
 def transform_application_forms():
-    with open(base_path / "applicationForms.json", "r", encoding="utf-8") as f:
+    with open(base_path / "applicationForms.json", encoding="utf-8") as f:
         application_form_dict = json.load(f)
 
     mappings = []
@@ -396,7 +396,7 @@ def optimize_rag():
         num_threads=2,
         track_stats=True,
         use_merge=False,
-        reflection_lm=lm, # TODO: use a separate reflection LLM
+        reflection_lm=lm,  # TODO: use a separate reflection LLM
     )
 
     rag = RAG()
@@ -414,7 +414,7 @@ def optimize_rag():
     optimized_rag = optimizer.compile(
         rag,
         trainset=trainset_examples,
-        valset=trainset_examples, # TODO: use a separate validation set
+        valset=trainset_examples,  # TODO: use a separate validation set
     )
 
     optimized_rag.save(base_path / "optimized_rag.json")

standalone/pdc-process-base-fields

@@ -69,8 +69,8 @@ those columns can simply do so.
   }
 """
 
-import json
 import argparse
+import json
 import sys
 
 try:
@@ -107,14 +107,18 @@ def main():
         # here would be RawDescriptionHelpFormatter -- I don't usually
         # use that because it can do some minor reformatting that I
         # find aesthetically displeasing, but hey, YMMV.
-        formatter_class=argparse.RawTextHelpFormatter)
-    allowed_formats = ["csv",]  # maybe also "markdown", "html", etc some day
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
+    allowed_formats = [
+        "csv",
+    ]  # maybe also "markdown", "html", etc some day
     arg_parser.add_argument(
-        '--output-format',
+        "--output-format",
         choices=allowed_formats,
         type=str.lower,
         default="csv",
-        help="Output format to show base fields in.  Default: 'csv'")
+        help="Output format to show base fields in.  Default: 'csv'",
+    )
 
     # Parse command line.
     args = arg_parser.parse_args()
@@ -123,11 +127,12 @@ def main():
         json_input = json.load(sys.stdin)
         parsed_input = pandas.json_normalize(json_input)
         # Switch to index=True to include a first column that shows DataFrame index
-        parsed_input.to_csv(sys.stdout, index=False, encoding='utf-8')
+        parsed_input.to_csv(sys.stdout, index=False, encoding="utf-8")
     else:  # can't happen
         raise ValueError(
-            f"ERROR: Unknown output format 'f{args.output_format}' requested")
+            f"ERROR: Unknown output format 'f{args.output_format}' requested"
+        )
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     main()

standalone/ruff.toml

@@ -0,0 +1,22 @@
+# Ruff configuration for the standalone Python scripts.
+#
+# The scripts deliberately carry no .py extension (they are standalone
+# command-line programs), so CI passes their paths to ruff and bandit
+# explicitly; a directory scan would silently skip them.
+target-version = "py311"
+
+[lint]
+select = [
+    "B",   # flake8-bugbear
+    "C4",  # flake8-comprehensions
+    "E4",  # pycodestyle imports
+    "E7",  # pycodestyle statements
+    "E9",  # pycodestyle runtime
+    "F",   # pyflakes
+    "I",   # isort
+    "SIM", # flake8-simplify
+    "UP",  # pyupgrade
+    "W",   # pycodestyle whitespace
+]
+# E501 (line-too-long) is not selected: the scripts embed long example
+# command lines and URLs in their docstrings on purpose.