deployment pattern updates

.dockerignore

@@ -11,6 +11,10 @@
 # endpoints, identifiers, or other data and are not reproducible inputs.
 nifi/conf/flow.json.gz
 nifi/conf/archive/
+nifi/user_scripts/logs/
+nifi/user_scripts/db/
+nifi/user_scripts/extensions/
+nifi/**/.venv/
 
 # Development and operating-system artefacts within the included source tree.
 nifi/**/__pycache__/

.gitattributes

@@ -70,5 +70,3 @@
 #*.exe filter=lfs diff=lfs merge=lfs -text
 # Doc images
 #docs/_static/img filter=lfs diff=lfs merge=lfs -text
-#services/cogstack-cohort/cui_pt2ch.json filter=lfs diff=lfs merge=lfs -text
-#services/cogstack-cohort/snomed_terms.json filter=lfs diff=lfs merge=lfs -text

.github/workflows/compose_config.yml

@@ -6,14 +6,70 @@ on:
     paths:
       - "deploy/*.env"
       - "deploy/export_env_vars.sh"
+      - "deploy/config_files.sh"
+      - "deploy/Makefile"
+      - "security/scripts/**"
+      - "scripts/opensearch-ai-init.sh"
+      - "scripts/tests/check_nifi_resources.py"
+      - "scripts/tests/test_configuration.py"
+      - "scripts/tests/test_nifi_runtime.py"
+      - "scripts/tests/check_opensearch_settings.py"
+      - "scripts/tests/test_opensearch_deployment.py"
+      - "services/elasticsearch/**"
+      - "services/cogstack-db/**"
+      - "services/pgsamples/**"
+      - "scripts/tests/test_database_deployment_security.py"
+      - "scripts/tests/test_postgres_certificates.py"
+      - "scripts/tests/test_postgres_initialization.py"
+      - "scripts/tests/test_mssql_initialization.py"
+      - "services/kibana/config/opensearch.yml"
+      - "security/templates/**"
+      - "security/es_roles/opensearch/**"
+      - "scripts/tests/test_opensearch_ai_init.py"
       - "deploy/services*.yml"
+      - "deploy/charts/**"
+      - "deploy/helm/**"
+      - "nifi/Dockerfile"
+      - "nifi/docker/**"
+      - "services/nginx/Dockerfile"
+      - "services/ollama-openai-proxy/Dockerfile"
+      - "scripts/tests/check_deployment_images.py"
+      - "scripts/tests/test_deployment_images.py"
       - "security/env/*.env"
       - ".github/workflows/compose_config.yml"
   pull_request:
     paths:
       - "deploy/*.env"
       - "deploy/export_env_vars.sh"
+      - "deploy/config_files.sh"
+      - "deploy/Makefile"
+      - "security/scripts/**"
+      - "scripts/opensearch-ai-init.sh"
+      - "scripts/tests/check_nifi_resources.py"
+      - "scripts/tests/test_configuration.py"
+      - "scripts/tests/test_nifi_runtime.py"
+      - "scripts/tests/check_opensearch_settings.py"
+      - "scripts/tests/test_opensearch_deployment.py"
+      - "services/elasticsearch/**"
+      - "services/cogstack-db/**"
+      - "services/pgsamples/**"
+      - "scripts/tests/test_database_deployment_security.py"
+      - "scripts/tests/test_postgres_certificates.py"
+      - "scripts/tests/test_postgres_initialization.py"
+      - "scripts/tests/test_mssql_initialization.py"
+      - "services/kibana/config/opensearch.yml"
+      - "security/templates/**"
+      - "security/es_roles/opensearch/**"
+      - "scripts/tests/test_opensearch_ai_init.py"
       - "deploy/services*.yml"
+      - "deploy/charts/**"
+      - "deploy/helm/**"
+      - "nifi/Dockerfile"
+      - "nifi/docker/**"
+      - "services/nginx/Dockerfile"
+      - "services/ollama-openai-proxy/Dockerfile"
+      - "scripts/tests/check_deployment_images.py"
+      - "scripts/tests/test_deployment_images.py"
       - "security/env/*.env"
       - ".github/workflows/compose_config.yml"
   workflow_dispatch:
@@ -32,10 +88,50 @@ jobs:
     steps:
       - uses: actions/checkout@v7
 
+      - name: Set up Python
+        uses: actions/setup-python@v7
+        with:
+          python-version: "3.11"
+
+      - name: Set up Java for certificate generation tests
+        uses: actions/setup-java@v6
+        with:
+          distribution: temurin
+          java-version: '21'
+
+      - name: Install deployment test dependency
+        run: python3 -m pip install PyYAML==6.0.3
+
+      - name: Set up Helm
+        uses: azure/setup-helm@v5.0.1
+        with:
+          version: v4.3.1
+
       - name: Validate Compose files
         shell: bash
         run: |
-          set -euo pipefail
-          source deploy/export_env_vars.sh
-          docker compose -f deploy/services.yml config --quiet
-          docker compose -f deploy/services.dev.yml config --quiet
+          make -C deploy check-config
+
+      - name: Verify deployment image pins
+        run: python3 -B scripts/tests/check_deployment_images.py --helm
+
+      - name: Test image drift checks
+        run: python3 -B -m unittest discover -s scripts/tests -p test_deployment_images.py -v
+
+      - name: Test configuration profiles and resource guards
+        run: python3 -B -m unittest discover -s scripts/tests -p test_configuration.py -v
+
+      - name: Test NiFi upstream settings, persistence and credential isolation
+        run: python3 -B scripts/tests/test_nifi_runtime.py -v
+
+      - name: Test database bootstrap and credential isolation
+        run: |
+          python3 -B -m unittest discover -s scripts/tests -p 'test_*initialization.py' -v
+          python3 -B scripts/tests/test_database_deployment_security.py
+          python3 -B scripts/tests/test_postgres_certificates.py
+
+      - name: Test OpenSearch AI configuration
+        run: python3 -B -m unittest discover -s scripts/tests -p test_opensearch_ai_init.py -v
+
+      - name: Test OpenSearch Compose and Helm compatibility
+        run: python3 -B -m unittest discover -s scripts/tests -p test_opensearch_deployment.py -v

.github/workflows/docker_elasticsearch_stack.yml

@@ -4,7 +4,16 @@ on:
   push:
     branches: ['**']
     tags: ['v*.*.*']
+    paths: &paths
+      - ".github/workflows/docker_elasticsearch_stack.yml"
+      - ".github/workflows/_docker_template.yml"
+      - ".gitattributes"
+      - "services/elasticsearch/**"
+      - "services/kibana/**"
+      - "!**/*.md"
+      - "!**/README*"
   pull_request:
+    paths: *paths
   release:
     types: [published]
   workflow_dispatch:

.github/workflows/docker_nifi.yml

@@ -4,7 +4,18 @@ on:
   push:
     branches: ['**']
     tags: ['v*.*.*']
+    paths: &paths
+      - ".github/workflows/docker_nifi.yml"
+      - ".github/workflows/_docker_template.yml"
+      - ".gitattributes"
+      - ".dockerignore"
+      - "nifi/**"
+      - "pyproject.toml"
+      - "uv.lock"
+      - "!**/*.md"
+      - "!**/README*"
   pull_request:
+    paths: *paths
   release:
     types: [published]
   workflow_dispatch:

.github/workflows/docker_nifi_nginx.yml

@@ -4,7 +4,15 @@ on:
   push:
     branches: ['**']
     tags: ['v*.*.*']
+    paths: &paths
+      - ".github/workflows/docker_nifi_nginx.yml"
+      - ".github/workflows/_docker_template.yml"
+      - ".gitattributes"
+      - "services/nginx/**"
+      - "!**/*.md"
+      - "!**/README*"
   pull_request:
+    paths: *paths
   release:
     types: [published]
   workflow_dispatch:

.github/workflows/smoke_elastic_services.yml

@@ -1,9 +1,29 @@
 name: smoke-elastic-services
 
 on:
-  pull_request:
   push:
     branches: ['**']
+    paths: &paths
+      - ".github/workflows/smoke_elastic_services.yml"
+      - ".gitattributes"
+      - "deploy/Makefile"
+      - "deploy/export_env_vars.sh"
+      - "deploy/config_files.sh"
+      - "deploy/*.env"
+      - "security/env/*.env"
+      - "security/scripts/**"
+      - "security/templates/**"
+      - "deploy/services.yml"
+      - "services/elasticsearch/**"
+      - "services/kibana/**"
+      - "security/es_roles/**"
+      - "scripts/tests/check_opensearch_settings.py"
+      - "scripts/tests/smoke_elastic_services.sh"
+      - "scripts/tests/smoke_http_checks.sh"
+      - "!**/*.md"
+      - "!**/README*"
+  pull_request:
+    paths: *paths
   workflow_dispatch:
 
 concurrency:

.github/workflows/smoke_gitea_service.yml

@@ -1,9 +1,26 @@
 name: smoke-gitea-service
 
 on:
-  pull_request:
   push:
     branches: ['**']
+    paths: &paths
+      - ".github/workflows/smoke_gitea_service.yml"
+      - ".gitattributes"
+      - "deploy/Makefile"
+      - "deploy/export_env_vars.sh"
+      - "deploy/config_files.sh"
+      - "deploy/*.env"
+      - "security/env/*.env"
+      - "security/scripts/**"
+      - "security/templates/**"
+      - "deploy/services.yml"
+      - "services/gitea/**"
+      - "scripts/tests/smoke_gitea_service.sh"
+      - "scripts/tests/smoke_http_checks.sh"
+      - "!**/*.md"
+      - "!**/README*"
+  pull_request:
+    paths: *paths
   workflow_dispatch:
 
 concurrency:

.github/workflows/smoke_nifi_services.yml

@@ -1,9 +1,33 @@
 name: smoke-nifi-services
 
 on:
-  pull_request:
   push:
     branches: ['**']
+    paths: &paths
+      - ".github/workflows/smoke_nifi_services.yml"
+      - ".gitattributes"
+      - "deploy/Makefile"
+      - "deploy/export_env_vars.sh"
+      - "deploy/config_files.sh"
+      - "deploy/*.env"
+      - "security/env/*.env"
+      - "security/scripts/**"
+      - "security/templates/**"
+      - ".dockerignore"
+      - "nifi/**"
+      - "pyproject.toml"
+      - "uv.lock"
+      - "services/nginx/**"
+      - "deploy/services.yml"
+      - "deploy/services.dev.yml"
+      - "scripts/tests/check_nifi_resources.py"
+      - "scripts/tests/smoke_nifi_services.sh"
+      - "scripts/tests/test_nifi_runtime.py"
+      - "scripts/tests/smoke_http_checks.sh"
+      - "!**/*.md"
+      - "!**/README*"
+  pull_request:
+    paths: *paths
   workflow_dispatch:
 
 concurrency:
@@ -28,11 +52,15 @@ jobs:
           java-version: '21'
 
       - name: Generate certificates
-        working-directory: security/scripts
         run: |
           set -euo pipefail
-          ./create_root_ca_cert.sh
-          ./create_nifi_certs.sh
+          make -C deploy init-security-nifi
+
+      - name: Validate published nginx startup
+        run: |
+          set -euo pipefail
+          source deploy/export_env_vars.sh
+          docker compose -f deploy/services.yml run --rm --no-deps nifi-nginx nginx -t
 
       - name: Smoke tests
         run: |
@@ -44,7 +72,6 @@ jobs:
         run: |
           set -euo pipefail
           source deploy/export_env_vars.sh
-          set -euo pipefail
           docker compose -f deploy/services.dev.yml ps
           docker compose -f deploy/services.dev.yml logs --no-color nifi nifi-nginx
 
@@ -53,5 +80,4 @@ jobs:
         run: |
           set -euo pipefail
           source deploy/export_env_vars.sh
-          set -euo pipefail
           docker compose -f deploy/services.dev.yml down -v

.github/workflows/smoke_postgres_services.yml

@@ -1,9 +1,26 @@
 name: smoke-postgres-services
 
 on:
-  pull_request:
   push:
     branches: ['**']
+    paths: &paths
+      - ".github/workflows/smoke_postgres_services.yml"
+      - ".gitattributes"
+      - "deploy/Makefile"
+      - "deploy/export_env_vars.sh"
+      - "deploy/config_files.sh"
+      - "deploy/*.env"
+      - "security/env/*.env"
+      - "security/scripts/**"
+      - "security/templates/**"
+      - "deploy/services.yml"
+      - "services/pgsamples"
+      - "services/pgsamples/**"
+      - "scripts/tests/smoke_postgres_services.sh"
+      - "!**/*.md"
+      - "!**/README*"
+  pull_request:
+    paths: *paths
   workflow_dispatch:
 
 concurrency:
@@ -21,6 +38,17 @@ jobs:
     steps:
       - uses: actions/checkout@v7
 
+      - name: Set up Java
+        uses: actions/setup-java@v6
+        with:
+          distribution: 'temurin'
+          java-version: '21'
+
+      - name: Generate PostgreSQL certificates
+        run: |
+          set -euo pipefail
+          make -C deploy init-security-postgresql
+
       - name: Start samples database
         run: |
           set -euo pipefail