Add ruff and bandit Python checks to CI

.github/workflows/ci.yml

@@ -64,6 +64,31 @@ jobs:
       - run: npm ci
       - run: npm run lint:tsc
 
+  # The standalone scripts carry no .py extension on purpose, so a
+  # directory scan would silently skip them. The shell expands the
+  # pdc-* globs below instead, so new scripts following the repo's
+  # pdc- naming convention are picked up without touching this file.
+  ruff:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+      - uses: actions/setup-python@v7
+        with:
+          python-version: "3.13"
+      - run: pip install ruff==0.17.4
+      - run: ruff check standalone/pdc-*
+      - run: ruff format --check standalone/pdc-*
+
+  bandit:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+      - uses: actions/setup-python@v7
+        with:
+          python-version: "3.13"
+      - run: pip install bandit==1.9.4
+      - run: bandit -q standalone/pdc-*
+
   build:
     runs-on: ubuntu-latest
     steps:

README.md

@@ -48,3 +48,18 @@ The `.env` file and `config.*.json` files are ignored by git,
 as they likely contain credentials and should not be committed.
 
 Finally, every option can be specified on the command line itself.
+
+## Standalone Python scripts
+
+The Python scripts in `standalone/` carry no filename extension, so
+a directory scan would silently skip them. CI lints them with ruff
+and bandit by shell-expanding the `standalone/pdc-*` glob, so for a
+new script to be covered by lint, its name should start with `pdc-`.
+
+To run the same checks locally:
+
+```
+ruff check standalone/pdc-*
+ruff format --check standalone/pdc-*
+bandit -q standalone/pdc-*
+```

standalone/pdc-optimize-field-matching

@@ -138,6 +138,7 @@ def retrieve_pdc_data(endpoint: str, *, params: dict[str, Any] | None = None):
         f"{DS_PDC_API_BASE_URL}{endpoint}",
         headers={"Authorization": f"Bearer {access_token}"},
         params=params or {},
+        timeout=60,
     )
 
     pdc_data_dict = pdc_data_response.json()
@@ -231,7 +232,6 @@ class RAG(dspy.Module):
         )
 
 
-
 @app.command()
 def retrieve_application_forms():
     retrieve_pdc_data(
@@ -396,7 +396,7 @@ def optimize_rag():
         num_threads=2,
         track_stats=True,
         use_merge=False,
-        reflection_lm=lm, # TODO: use a separate reflection LLM
+        reflection_lm=lm,  # TODO: use a separate reflection LLM
     )
 
     rag = RAG()
@@ -414,7 +414,7 @@ def optimize_rag():
     optimized_rag = optimizer.compile(
         rag,
         trainset=trainset_examples,
-        valset=trainset_examples, # TODO: use a separate validation set
+        valset=trainset_examples,  # TODO: use a separate validation set
     )
 
     optimized_rag.save(base_path / "optimized_rag.json")

standalone/pdc-process-base-fields

@@ -69,8 +69,8 @@ those columns can simply do so.
   }
 """
 
-import json
 import argparse
+import json
 import sys
 
 try:
@@ -107,14 +107,18 @@ def main():
         # here would be RawDescriptionHelpFormatter -- I don't usually
         # use that because it can do some minor reformatting that I
         # find aesthetically displeasing, but hey, YMMV.
-        formatter_class=argparse.RawTextHelpFormatter)
-    allowed_formats = ["csv",]  # maybe also "markdown", "html", etc some day
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
+    allowed_formats = [
+        "csv",
+    ]  # maybe also "markdown", "html", etc some day
     arg_parser.add_argument(
-        '--output-format',
+        "--output-format",
         choices=allowed_formats,
         type=str.lower,
         default="csv",
-        help="Output format to show base fields in.  Default: 'csv'")
+        help="Output format to show base fields in.  Default: 'csv'",
+    )
 
     # Parse command line.
     args = arg_parser.parse_args()
@@ -123,11 +127,12 @@ def main():
         json_input = json.load(sys.stdin)
         parsed_input = pandas.json_normalize(json_input)
         # Switch to index=True to include a first column that shows DataFrame index
-        parsed_input.to_csv(sys.stdout, index=False, encoding='utf-8')
+        parsed_input.to_csv(sys.stdout, index=False, encoding="utf-8")
     else:  # can't happen
         raise ValueError(
-            f"ERROR: Unknown output format 'f{args.output_format}' requested")
+            f"ERROR: Unknown output format 'f{args.output_format}' requested"
+        )
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     main()