From 935e48215e4e2f708bc2c74bea29e3f6d7f3599f Mon Sep 17 00:00:00 2001 From: vladd-bit Date: Wed, 4 Mar 2026 12:31:31 +0000 Subject: [PATCH 01/28] Docs: updated ES/OS setup docs. --- docs/deploy/services.md | 14 ++++++++++++- docs/security/elasticsearch_opensearch.md | 24 ++++++++++++++++++++++- 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/docs/deploy/services.md b/docs/deploy/services.md index 288290cf1..3874707a7 100644 --- a/docs/deploy/services.md +++ b/docs/deploy/services.md @@ -410,11 +410,23 @@ Web UI for exploring indexed data, visualising documents, managing index templat - URL: **https://localhost:5601** -#### credentials +#### Credentials - **OpenSearch Dashboards:** `admin` / `admin` - **Elasticsearch Native:** `elastic` / `kibanaserver` +#### First login setup (OpenSearch Dashboards) + +After signing in to OpenSearch Dashboards: + +1. Open the user menu and choose **Switch tenant**. +2. Select `Global` so saved objects are shared across users/roles. +3. Go to **Stack Management** → **Workspaces** and click **Create workspace**. +4. Create a workspace (for example `cogstack-main`) in the `Global` tenant. +5. Create data views/dashboards in that workspace. + +`Private` tenant is isolated per user and should be avoided for shared environments. + #### Containers - `cogstack-kibana` (OpenSearch Dashboards or Kibana depending on configuration) diff --git a/docs/security/elasticsearch_opensearch.md b/docs/security/elasticsearch_opensearch.md index 90f964669..65a3331a3 100644 --- a/docs/security/elasticsearch_opensearch.md +++ b/docs/security/elasticsearch_opensearch.md @@ -145,7 +145,7 @@ security/certificates/elastic/opensearch/ | **Kibana** | `elasticsearch-{1,2,3}.crt`, `elasticsearch-{1,2,3}.key`, `elastic-stack-ca.crt.pem` | `security/certificates/elastic/elasticsearch/` | | **OpenDashboard (OpenSearch)** | `admin.pem`, `admin-key.pem`, `es_kibana_client.pem`, `es_kibana_client.key` | `security/certificates/elastic/opensearch/` | -All certificate references in `services/kibana/config/kibana_opensearch.yml` or `services.yml` must point to these locations. +All certificate references in `services/kibana/config/opensearch.yml` or `services.yml` must point to these locations. --- @@ -218,6 +218,28 @@ Troubleshooting: OpenSearch includes default roles (`admin`, `kibanaserver`, `readall`, `snapshotrestore`, etc.) — always change their passwords after first run. +##### OpenSearch Dashboards post-login setup (Global tenant + workspace) + +After your first login to `https://localhost:5601` (default: `admin` / `admin`): + +1. Open the user menu (top-right) and select **Switch tenant**. +2. Select `Global`, then apply the change. +3. Open **Stack Management** → **Workspaces** (or the workspace switcher in the header) and click **Create workspace**. +4. Enter a workspace name (for example `cogstack-main`) and keep it in the `Global` tenant. +5. Save, then create data views and dashboards inside this workspace. + +Use `Global` for shared dashboards and saved objects. +`Private` tenant content is user-specific and not visible to other users. + +If tenant switching or workspace creation is not available in the UI, verify these settings in `services/kibana/config/opensearch.yml`: + +```yaml +opensearch_security.multitenancy.enabled: true +opensearch_security.multitenancy.tenants.enable_global: true +opensearch_security.multitenancy.tenants.preferred: ["Global"] +workspace.enabled: true +``` + --- #### Elasticsearch (native) From fefb1e1d00b2fafde2f9c696a03be68ef85e18c1 Mon Sep 17 00:00:00 2001 From: vladd-bit Date: Wed, 22 Apr 2026 09:06:58 +0100 Subject: [PATCH 02/28] NiFi: fixed localhost url issues after 2.9 update. --- deploy/nifi.env | 5 +++-- nifi/Dockerfile | 6 +++--- services/nginx/config/nginx.conf.template | 7 +++++++ 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/deploy/nifi.env b/deploy/nifi.env index 7dedcbfd4..4fdc3fcf3 100644 --- a/deploy/nifi.env +++ b/deploy/nifi.env @@ -44,8 +44,9 @@ NIFI_WEB_PROXY_CONTEXT_PATH="/nifi" # if you wish to run NiFi on another port, this goes for both NIFI_WEB_PROXY_HOST and NIFI_INTERNAL_PORT # you will also need to change the NGINX configuration -# if you wish to -NIFI_WEB_PROXY_HOST="localhost:8443" +# If browsers have trouble with localhost-scoped auth cookies, using 127.0.0.1 +# avoids the special-case cookie handling around localhost. +NIFI_WEB_PROXY_HOST="localhost:8443,127.0.0.1:8443" NIFI_INTERNAL_PORT=8443 diff --git a/nifi/Dockerfile b/nifi/Dockerfile index 74cc912d3..a164ddd33 100644 --- a/nifi/Dockerfile +++ b/nifi/Dockerfile @@ -4,7 +4,7 @@ FROM apache/nifi:${NIFI_VERSION} SHELL ["/bin/bash", "-o", "pipefail", "-c"] -ARG NIFI_VERSION=2.8.0 +ARG NIFI_VERSION=2.9.0 ARG GROOVY_VERSION=5.0.4 ARG HTTP_PROXY="" @@ -149,8 +149,8 @@ RUN $GROOVY_BIN/grape -V install org.apache.avro avro 1.12.0 # INSTALL NAR extensions WORKDIR /opt/nifi/nifi-current/lib/ -# --- Hotfix: NiFi 2.7.2 + Azure + Netty QUIC missing class (netty 4.2.9.Final) --- -ARG NETTY_VERSION=4.2.9.Final +# --- Hotfix: NiFi 2.7.2 + Azure + Netty QUIC missing class (netty 4.2.12.Final) --- +ARG NETTY_VERSION=4.2.12.Final RUN set -eux; \ cd /opt/nifi/nifi-current/lib; \ curl -fL --retry 5 --retry-delay 2 \ diff --git a/services/nginx/config/nginx.conf.template b/services/nginx/config/nginx.conf.template index a046069a9..b80ba73ea 100644 --- a/services/nginx/config/nginx.conf.template +++ b/services/nginx/config/nginx.conf.template @@ -83,6 +83,13 @@ http { proxy_ssl_server_name on; + # Firefox has trouble with the localhost-scoped NiFi auth cookie. + # Redirect localhost to 127.0.0.1 so the browser gets a stable IP-scoped + # cookie domain instead. + if ($host = localhost) { + return 308 https://127.0.0.1:8443$request_uri; + } + ssl_certificate /certificates/nifi/nifi.pem; ssl_certificate_key /certificates/nifi/nifi.key; ssl_client_certificate /certificates/root/root-ca.pem; From da4a9b939f4de10cfd6832635e8cab624a89727d Mon Sep 17 00:00:00 2001 From: vladd-bit Date: Wed, 24 Jun 2026 16:17:32 +0100 Subject: [PATCH 03/28] Update submodules to latest release tags (or default branch) --- services/cogstack-jupyter-hub | 2 +- services/cogstack-nlp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/services/cogstack-jupyter-hub b/services/cogstack-jupyter-hub index 0362e4bce..ea5a68511 160000 --- a/services/cogstack-jupyter-hub +++ b/services/cogstack-jupyter-hub @@ -1 +1 @@ -Subproject commit 0362e4bce262fa72a0b270d4d1a85e76e9109ab0 +Subproject commit ea5a68511702553d7f1ccac41939919fa38b6d86 diff --git a/services/cogstack-nlp b/services/cogstack-nlp index 14cb02e2c..7b193624e 160000 --- a/services/cogstack-nlp +++ b/services/cogstack-nlp @@ -1 +1 @@ -Subproject commit 14cb02e2c8b8b188cd12f727a20991a3097a6537 +Subproject commit 7b193624e10aa550fec44890b76a4c84cdc65897 From 659fc29297a115e052b4967afb0d25bd9dcaed99 Mon Sep 17 00:00:00 2001 From: vladd-bit Date: Thu, 22 Oct 2026 11:35:33 +0100 Subject: [PATCH 04/28] Removed old version of CogStack cohort. --- .gitattributes | 2 -- .gitmodules | 4 ---- deploy/services.yml | 19 ------------------- docs/deploy/services.md | 1 - services/cogstack-cohort | 1 - 5 files changed, 27 deletions(-) delete mode 160000 services/cogstack-cohort diff --git a/.gitattributes b/.gitattributes index c7b30618e..95bfadaea 100644 --- a/.gitattributes +++ b/.gitattributes @@ -70,5 +70,3 @@ #*.exe filter=lfs diff=lfs merge=lfs -text # Doc images #docs/_static/img filter=lfs diff=lfs merge=lfs -text -#services/cogstack-cohort/cui_pt2ch.json filter=lfs diff=lfs merge=lfs -text -#services/cogstack-cohort/snomed_terms.json filter=lfs diff=lfs merge=lfs -text diff --git a/.gitmodules b/.gitmodules index 97ca4774e..a012ead02 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,7 +1,3 @@ -[submodule "services/cogstack-cohort"] - path = services/cogstack-cohort - url = https://github.com/CogStack/CogStackCohort.git - ignore = all [submodule "services/cogstack-nlp"] path = services/cogstack-nlp url = https://github.com/CogStack/cogstack-nlp.git diff --git a/deploy/services.yml b/deploy/services.yml index 2fab470da..c82c6cb3f 100644 --- a/deploy/services.yml +++ b/deploy/services.yml @@ -678,25 +678,6 @@ services: extra_hosts: *common-hosts logging: *nifi-logging-common -#---------------------------------------------------------------------------# -# Cogstack cohort service # -#---------------------------------------------------------------------------# - cogstack-cohort: - container_name: cogstack-cohort - image: cogstacksystems/cogstackcohort:latest - shm_size: ${DOCKER_SHM_SIZE:-"1g"} - restart: always - env_file: *all-env - volumes: - - ./../data/cogstack-cohort:/usr/src/app/server/data:rw - - ./../services/cogstack-cohort/cui_pt2ch.json:/usr/src/app/server/data/cui_pt2ch.json:rw - - ./../services/cogstack-cohort/snomed_terms.json:/usr/src/app/server/data/snomed_terms.json:rw - networks: - - cognet - ports: - - "3001:3000" - logging: *logging-common - #---------------------------------------------------------------------------# # Version control service # #---------------------------------------------------------------------------# diff --git a/docs/deploy/services.md b/docs/deploy/services.md index a3b40e3fb..6c7835467 100644 --- a/docs/deploy/services.md +++ b/docs/deploy/services.md @@ -26,7 +26,6 @@ The core services defined in `services.yml` include: - **nifi-nginx** — Reverse proxy providing secure access to NiFi. - **kibana** — Kibana or OpenSearch Dashboards, depending on the backend. - **gitea** — Self-hosted Git service. -- **cogstack-cohort** — Cohort discovery interface. Additional services, including MedCAT, MedCAT Trainer, OCR, and JupyterHub, are started from Compose files in their respective `services/` directories. diff --git a/services/cogstack-cohort b/services/cogstack-cohort deleted file mode 160000 index 88fa2a107..000000000 --- a/services/cogstack-cohort +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 88fa2a1071e76964c1ed97f9443279352068afc6 From 914c4cbf89f948b8a2b25553c11cea92c076fea4 Mon Sep 17 00:00:00 2001 From: vladd-bit Date: Thu, 22 Oct 2026 12:39:20 +0100 Subject: [PATCH 05/28] Deploy: OpenSearch updated to vsersion 3.9. --- deploy/elasticsearch.env | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/deploy/elasticsearch.env b/deploy/elasticsearch.env index 36d1054a0..e5a3ac1c0 100644 --- a/deploy/elasticsearch.env +++ b/deploy/elasticsearch.env @@ -9,10 +9,10 @@ ELASTICSEARCH_VERSION=opensearch # possible values : # - elasticsearch : docker.elastic.co/elasticsearch/elasticsearch:9.5.2 # - elasticsearch (custom cogstack image) : cogstacksystems/cogstack-elasticsearch:latest -# - opensearch : opensearchproject/opensearch:3.8.0 +# - opensearch : opensearchproject/opensearch:3.9.0 # the custom cogstack image is always based on the last image of ES native -ELASTICSEARCH_DOCKER_IMAGE=opensearchproject/opensearch:3.8.0 +ELASTICSEARCH_DOCKER_IMAGE=opensearchproject/opensearch:3.9.0 ELASTICSEARCH_LOG_LEVEL=INFO @@ -216,10 +216,10 @@ KIBANA_CONFIG_FILE_VERSION=opensearch_dashboards # possible values: # - elasticsearch : docker.elastic.co/kibana/kibana:9.5.2 # - elasticsearch (custom cogstack image) : cogstacksystems/cogstack-kibana:latest -# - opensearch : opensearchproject/opensearch-dashboards:3.8.0 +# - opensearch : opensearchproject/opensearch-dashboards:3.9.0 # the custom cogstack image is always based on the last image of ES native -ELASTICSEARCH_KIBANA_DOCKER_IMAGE=opensearchproject/opensearch-dashboards:3.8.0 +ELASTICSEARCH_KIBANA_DOCKER_IMAGE=opensearchproject/opensearch-dashboards:3.9.0 KIBANA_SERVER_NAME="cogstack-kibana" KIBANA_PUBLIC_BASE_URL="https://elasticsearch-1:5601" From af76baf83d1e836574987c430931e8bd72806d09 Mon Sep 17 00:00:00 2001 From: vladd-bit Date: Fri, 23 Oct 2026 11:42:28 +0100 Subject: [PATCH 06/28] Deploy/Services: (ES) updated OpenSearch to 3.9.0 and made relevant changes to old opensearch configs. --- deploy/charts/opensearch/README.md | 6 +- .../opensearch/files/performance-analyzer | 1 + .../opensearch/files/search-entrypoint.sh | 1 + .../configmap-opensearch-config.yaml | 8 ++ .../templates/secret-credentials.yaml | 10 +++ .../templates/statefulset-opensearch.yaml | 62 +++++++++++---- deploy/charts/opensearch/values.yaml | 12 ++- deploy/elasticsearch.env | 6 +- deploy/services.yml | 78 ++++++++++++------- docs/security/elasticsearch_opensearch.md | 54 +++++++++++++ security/env/users_elasticsearch.env | 7 ++ services/elasticsearch/config/opensearch.yml | 29 +++---- .../performance-analyzer/agent-stats-metadata | 1 + .../performance-analyzer.properties | 9 +++ .../plugin-stats-metadata | 1 + services/elasticsearch/search-entrypoint.sh | 55 +++++++++++++ 16 files changed, 271 insertions(+), 69 deletions(-) create mode 120000 deploy/charts/opensearch/files/performance-analyzer create mode 120000 deploy/charts/opensearch/files/search-entrypoint.sh create mode 100644 services/elasticsearch/config/performance-analyzer/agent-stats-metadata create mode 100644 services/elasticsearch/config/performance-analyzer/performance-analyzer.properties create mode 100644 services/elasticsearch/config/performance-analyzer/plugin-stats-metadata create mode 100644 services/elasticsearch/search-entrypoint.sh diff --git a/deploy/charts/opensearch/README.md b/deploy/charts/opensearch/README.md index 0f4b6499c..afd81fcb5 100644 --- a/deploy/charts/opensearch/README.md +++ b/deploy/charts/opensearch/README.md @@ -20,6 +20,7 @@ Helm chart for deploying OpenSearch and/or OpenSearch Dashboards using the CogSt 2. Kubernetes Secrets containing TLS materials for enabled components. 3. If `credentials.create=false`, an existing Secret with: - `OPENSEARCH_INITIAL_ADMIN_PASSWORD` when `opensearch.enabled=true` + - `OPENSEARCH_COMPLIANCE_SALT` and `OPENSEARCH_DATASOURCE_MASTER_KEY` when `opensearch.enabled=true` - `KIBANA_USER` and `KIBANA_PASSWORD` when `dashboards.enabled=true` ## Required certificate secrets @@ -103,11 +104,14 @@ helm template cogstack-opensearch ./deploy/charts/opensearch \ - The standard install/render commands now use `-f ./deploy/helm/opensearch.values.yaml`; that file is for cluster-specific overrides only. - The shared `services/`, `security/`, and selected `deploy/` env files are consumed automatically by the chart defaults; you do not need to repeat those paths in the values file. - `envFile.raw` defaults to `deploy/elasticsearch.env` and can still be overridden; the chart reads only `ELASTICSEARCH_CLUSTER_NAME`, `ELASTICSEARCH_JAVA_OPTS` / `OPENSEARCH_JAVA_OPTS`, and `KIBANA_SERVER_NAME`, while pod IP and discovery hosts remain Kubernetes-specific. -- `usersEnvFile.raw` defaults to `security/env/users_elasticsearch.env` and can still be overridden; only the credential keys required by the enabled components are imported. +- `usersEnvFile.raw` defaults to `security/env/users_elasticsearch.env` and can still be overridden; only the credential keys required by the enabled components are imported. The bundled values are public development defaults. Provide a deployment-specific masking salt and datasource encryption key through this override or `credentials.existingSecret` in production, with identical values on every node. - `certificatesEnvFile.raw` defaults to `security/env/certificates_elasticsearch.env` and can still be overridden; currently `ES_CLIENT_CERT_NAME` is used to resolve Dashboards cert secret keys (`.pem` / `.key`). - `deploy/elasticsearch.env` shared values are used where they make sense on Kubernetes (`ELASTICSEARCH_CLUSTER_NAME`, `ELASTICSEARCH_JAVA_OPTS` / `OPENSEARCH_JAVA_OPTS`, `KIBANA_SERVER_NAME`), while pod IP and discovery hosts remain Kubernetes-specific. - By default, `certificates.opensearchNodeFiles[*]` maps pod ordinals `0/1/2` to repo-style node cert keys `elasticsearch-1/2/3`. - `opensearch.logPersistence` and `opensearch.performanceAnalyzerPersistence` default to PVC-backed storage to stay closer to the Docker Compose deployment. +- Performance Analyzer configuration and metadata files come from `services/elasticsearch/config/performance-analyzer/`. Its metrics database volume is mounted at `/var/lib/opensearch-performance-analyzer`; metadata paths are regular files mounted from the ConfigMap. +- The shared entrypoint skips the legacy Performance Analyzer agent when its RCA libraries are absent, as in the OpenSearch 3.9 image. The in-process collectors remain installed, but the legacy port 9600 API is unavailable without a compatible agent distribution. +- Deployment configuration and certificates are mounted under `config-input/`. The shared search entrypoint copies them into the image's runtime `config/` directory and sets OpenSearch directories to `0700` and files to `0600` before calling the original image entrypoint. Restart pods after updating inputs. The demo security installer is disabled because this chart supplies its own configuration. - `opensearch.snapshotBackups` adds shared PVC-backed mounts for `/mnt/es_data_backups` and `/mnt/es_config_backups`; use RWX storage or set `existingClaim` values, and still set `path.repo` in the shared OpenSearch config if you want the cluster to use them. - `configFiles.opensearchRaw` can be set from `services/elasticsearch/config/opensearch.yml`. - `configFiles.log4jRaw` can be set from `services/elasticsearch/config/log4j2_opensearch.properties`. diff --git a/deploy/charts/opensearch/files/performance-analyzer b/deploy/charts/opensearch/files/performance-analyzer new file mode 120000 index 000000000..d2ccabde1 --- /dev/null +++ b/deploy/charts/opensearch/files/performance-analyzer @@ -0,0 +1 @@ +../../../../services/elasticsearch/config/performance-analyzer \ No newline at end of file diff --git a/deploy/charts/opensearch/files/search-entrypoint.sh b/deploy/charts/opensearch/files/search-entrypoint.sh new file mode 120000 index 000000000..3c8d0bcec --- /dev/null +++ b/deploy/charts/opensearch/files/search-entrypoint.sh @@ -0,0 +1 @@ +../../../../services/elasticsearch/search-entrypoint.sh \ No newline at end of file diff --git a/deploy/charts/opensearch/templates/configmap-opensearch-config.yaml b/deploy/charts/opensearch/templates/configmap-opensearch-config.yaml index b05d2843d..d5cf56773 100644 --- a/deploy/charts/opensearch/templates/configmap-opensearch-config.yaml +++ b/deploy/charts/opensearch/templates/configmap-opensearch-config.yaml @@ -8,8 +8,16 @@ metadata: labels: {{- include "cogstack-opensearch.labels" . | nindent 4 }} data: + search-entrypoint.sh: | +{{ .Files.Get "files/search-entrypoint.sh" | indent 4 }} opensearch.yml: | {{ $opensearchConfigRaw | indent 4 }} log4j2.properties: | {{ $log4jRaw | indent 4 }} + performance-analyzer.properties: | +{{ .Files.Get "files/performance-analyzer/performance-analyzer.properties" | indent 4 }} + plugin-stats-metadata: | +{{ .Files.Get "files/performance-analyzer/plugin-stats-metadata" | indent 4 }} + agent-stats-metadata: | +{{ .Files.Get "files/performance-analyzer/agent-stats-metadata" | indent 4 }} {{- end }} diff --git a/deploy/charts/opensearch/templates/secret-credentials.yaml b/deploy/charts/opensearch/templates/secret-credentials.yaml index e4c320a4f..6a5facc6c 100644 --- a/deploy/charts/opensearch/templates/secret-credentials.yaml +++ b/deploy/charts/opensearch/templates/secret-credentials.yaml @@ -4,6 +4,14 @@ {{- if hasKey $usersData "OPENSEARCH_INITIAL_ADMIN_PASSWORD" -}} {{- $adminPassword = index $usersData "OPENSEARCH_INITIAL_ADMIN_PASSWORD" -}} {{- end -}} +{{- $complianceSalt := .Values.credentials.complianceSalt -}} +{{- if hasKey $usersData "OPENSEARCH_COMPLIANCE_SALT" -}} +{{- $complianceSalt = index $usersData "OPENSEARCH_COMPLIANCE_SALT" -}} +{{- end -}} +{{- $datasourceMasterKey := .Values.credentials.datasourceMasterKey -}} +{{- if hasKey $usersData "OPENSEARCH_DATASOURCE_MASTER_KEY" -}} +{{- $datasourceMasterKey = index $usersData "OPENSEARCH_DATASOURCE_MASTER_KEY" -}} +{{- end -}} {{- $kibanaUser := .Values.credentials.kibanaUser -}} {{- if hasKey $usersData "KIBANA_USER" -}} {{- $kibanaUser = index $usersData "KIBANA_USER" -}} @@ -22,6 +30,8 @@ type: Opaque stringData: {{- if .Values.opensearch.enabled }} OPENSEARCH_INITIAL_ADMIN_PASSWORD: {{ $adminPassword | quote }} + OPENSEARCH_COMPLIANCE_SALT: {{ required "Set OPENSEARCH_COMPLIANCE_SALT in usersEnvFile.raw or credentials.complianceSalt" $complianceSalt | quote }} + OPENSEARCH_DATASOURCE_MASTER_KEY: {{ required "Set OPENSEARCH_DATASOURCE_MASTER_KEY in usersEnvFile.raw or credentials.datasourceMasterKey" $datasourceMasterKey | quote }} {{- end }} {{- if .Values.dashboards.enabled }} KIBANA_USER: {{ $kibanaUser | quote }} diff --git a/deploy/charts/opensearch/templates/statefulset-opensearch.yaml b/deploy/charts/opensearch/templates/statefulset-opensearch.yaml index e7317786a..15dd3b280 100644 --- a/deploy/charts/opensearch/templates/statefulset-opensearch.yaml +++ b/deploy/charts/opensearch/templates/statefulset-opensearch.yaml @@ -80,6 +80,8 @@ spec: - name: opensearch image: "{{ .Values.opensearch.image.repository }}:{{ .Values.opensearch.image.tag }}" imagePullPolicy: {{ .Values.opensearch.image.pullPolicy }} + command: ["/bin/bash", "/opt/cogstack/search-entrypoint.sh"] + args: ["opensearch"] securityContext: {{- toYaml .Values.opensearch.securityContext | nindent 12 }} ports: @@ -120,6 +122,16 @@ spec: secretKeyRef: name: {{ include "cogstack-opensearch.credentialsSecretName" . }} key: OPENSEARCH_INITIAL_ADMIN_PASSWORD + - name: OPENSEARCH_COMPLIANCE_SALT + valueFrom: + secretKeyRef: + name: {{ include "cogstack-opensearch.credentialsSecretName" . }} + key: OPENSEARCH_COMPLIANCE_SALT + - name: OPENSEARCH_DATASOURCE_MASTER_KEY + valueFrom: + secretKeyRef: + name: {{ include "cogstack-opensearch.credentialsSecretName" . }} + key: OPENSEARCH_DATASOURCE_MASTER_KEY {{- with .Values.opensearch.extraEnv }} {{- toYaml . | nindent 12 }} {{- end }} @@ -141,63 +153,79 @@ spec: {{- toYaml .Values.opensearch.resources | nindent 12 }} volumeMounts: - name: opensearch-config - mountPath: /usr/share/opensearch/config/opensearch.yml + mountPath: /opt/cogstack/search-entrypoint.sh + subPath: search-entrypoint.sh + readOnly: true + - name: opensearch-config + mountPath: /usr/share/opensearch/config-input/opensearch.yml subPath: opensearch.yml - name: opensearch-config - mountPath: /usr/share/opensearch/config/log4j2.properties + mountPath: /usr/share/opensearch/config-input/log4j2.properties subPath: log4j2.properties + - name: opensearch-config + mountPath: /usr/share/opensearch/config-input/opensearch-performance-analyzer/performance-analyzer.properties + subPath: performance-analyzer.properties + readOnly: true + - name: opensearch-config + mountPath: /usr/share/opensearch/config-input/opensearch-performance-analyzer/plugin-stats-metadata + subPath: plugin-stats-metadata + readOnly: true + - name: opensearch-config + mountPath: /usr/share/opensearch/config-input/opensearch-performance-analyzer/agent-stats-metadata + subPath: agent-stats-metadata + readOnly: true - name: opensearch-security - mountPath: /usr/share/opensearch/config/opensearch-security/config.yml + mountPath: /usr/share/opensearch/config-input/opensearch-security/config.yml subPath: config.yml - name: opensearch-security - mountPath: /usr/share/opensearch/config/opensearch-security/internal_users.yml + mountPath: /usr/share/opensearch/config-input/opensearch-security/internal_users.yml subPath: internal_users.yml - name: opensearch-security - mountPath: /usr/share/opensearch/config/opensearch-security/roles.yml + mountPath: /usr/share/opensearch/config-input/opensearch-security/roles.yml subPath: roles.yml - name: opensearch-security - mountPath: /usr/share/opensearch/config/opensearch-security/roles_mapping.yml + mountPath: /usr/share/opensearch/config-input/opensearch-security/roles_mapping.yml subPath: roles_mapping.yml {{- if gt (len $nodeCertFiles) 0 }} - name: opensearch-certs-runtime - mountPath: /usr/share/opensearch/config/root-ca.crt + mountPath: /usr/share/opensearch/config-input/root-ca.crt subPath: root-ca.crt readOnly: true - name: opensearch-certs-runtime - mountPath: /usr/share/opensearch/config/esnode.crt + mountPath: /usr/share/opensearch/config-input/esnode.crt subPath: esnode.crt readOnly: true - name: opensearch-certs-runtime - mountPath: /usr/share/opensearch/config/esnode.key + mountPath: /usr/share/opensearch/config-input/esnode.key subPath: esnode.key readOnly: true - name: opensearch-certs-runtime - mountPath: /usr/share/opensearch/config/admin.crt + mountPath: /usr/share/opensearch/config-input/admin.crt subPath: admin.crt readOnly: true - name: opensearch-certs-runtime - mountPath: /usr/share/opensearch/config/admin.key.pem + mountPath: /usr/share/opensearch/config-input/admin.key.pem subPath: admin.key.pem readOnly: true {{- else }} - name: opensearch-certs - mountPath: /usr/share/opensearch/config/root-ca.crt + mountPath: /usr/share/opensearch/config-input/root-ca.crt subPath: {{ .Values.certificates.opensearchFiles.rootCA }} readOnly: true - name: opensearch-certs - mountPath: /usr/share/opensearch/config/esnode.crt + mountPath: /usr/share/opensearch/config-input/esnode.crt subPath: {{ .Values.certificates.opensearchFiles.nodeCert }} readOnly: true - name: opensearch-certs - mountPath: /usr/share/opensearch/config/esnode.key + mountPath: /usr/share/opensearch/config-input/esnode.key subPath: {{ .Values.certificates.opensearchFiles.nodeKey }} readOnly: true - name: opensearch-certs - mountPath: /usr/share/opensearch/config/admin.crt + mountPath: /usr/share/opensearch/config-input/admin.crt subPath: {{ .Values.certificates.opensearchFiles.adminCert }} readOnly: true - name: opensearch-certs - mountPath: /usr/share/opensearch/config/admin.key.pem + mountPath: /usr/share/opensearch/config-input/admin.key.pem subPath: {{ .Values.certificates.opensearchFiles.adminKey }} readOnly: true {{- end }} @@ -206,7 +234,7 @@ spec: - name: logs mountPath: /usr/share/opensearch/logs - name: performance-analyzer - mountPath: /usr/share/opensearch/config/opensearch-performance-analyzer/plugins-stats-metadata + mountPath: /var/lib/opensearch-performance-analyzer {{- if .Values.opensearch.snapshotBackups.enabled }} - name: snapshot-backup-data mountPath: {{ .Values.opensearch.snapshotBackups.data.mountPath }} diff --git a/deploy/charts/opensearch/values.yaml b/deploy/charts/opensearch/values.yaml index 53dd879d8..0418b2dba 100644 --- a/deploy/charts/opensearch/values.yaml +++ b/deploy/charts/opensearch/values.yaml @@ -32,6 +32,8 @@ usersEnvFile: # Only these keys are imported into the credentials Secret. includeKeys: - OPENSEARCH_INITIAL_ADMIN_PASSWORD + - OPENSEARCH_COMPLIANCE_SALT + - OPENSEARCH_DATASOURCE_MASTER_KEY - KIBANA_USER - KIBANA_PASSWORD @@ -52,14 +54,14 @@ opensearch: tag: "3.5.0" pullPolicy: IfNotPresent clusterName: elasticsearch-cogstack-cluster - javaOpts: "-Xms512m -Xmx512m -Des.failure_store_feature_flag_enabled=true" + javaOpts: "-Xms512m -Xmx512m -Des.failure_store_feature_flag_enabled=true --enable-native-access=ALL-UNNAMED" resources: requests: cpu: "500m" - memory: "1Gi" + memory: "2Gi" limits: cpu: "1" - memory: "1Gi" + memory: "2Gi" persistence: enabled: true accessModes: @@ -150,6 +152,10 @@ credentials: create: true existingSecret: "" adminPassword: "Kibanaserver!1" + # Required when usersEnvFile.raw overrides the bundled development values. + # Use deployment-specific values or credentials.existingSecret in production. + complianceSalt: "" + datasourceMasterKey: "" kibanaUser: kibanaserver kibanaPassword: kibanaserver diff --git a/deploy/elasticsearch.env b/deploy/elasticsearch.env index e5a3ac1c0..ed9186166 100644 --- a/deploy/elasticsearch.env +++ b/deploy/elasticsearch.env @@ -93,13 +93,15 @@ ELASTICSEARCH_BACKUP_PARTITION_CONFIG=../data/es_snapshot_backups/config_backup ELASTICSEARCH_SECURITY_DIR=../security/certificates/elastic/ # MEMORY CONFIG -ELASTICSEARCH_JAVA_OPTS="-Xms512m -Xmx512m -Des.failure_store_feature_flag_enabled=true" +ELASTICSEARCH_JAVA_OPTS="-Xms512m -Xmx512m -Des.failure_store_feature_flag_enabled=true --enable-native-access=ALL-UNNAMED" ES_JAVA_OPTS=$ELASTICSEARCH_JAVA_OPTS OPENSEARCH_JAVA_OPTS=$ELASTICSEARCH_JAVA_OPTS ELASTICSEARCH_DOCKER_CPU_MIN=1 ELASTICSEARCH_DOCKER_CPU_MAX=1 -ELASTICSEARCH_DOCKER_RAM=1g +# Leave room beyond the Java heap for direct buffers, native code, and plugins. +# OpenSearch 3.9 exceeds a 1 GiB container limit during plugin initialization. +ELASTICSEARCH_DOCKER_RAM=2g ELASTICSEARCH_DOCKER_SHM_SIZE=512m diff --git a/deploy/services.yml b/deploy/services.yml index c82c6cb3f..194cf138e 100644 --- a/deploy/services.yml +++ b/deploy/services.yml @@ -128,20 +128,25 @@ x-db-common: &db-common x-es-common-volumes: &es-common-volumes # Shared configs - - ../services/elasticsearch/config/${ELASTICSEARCH_VERSION:-opensearch}.yml:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/config/${ELASTICSEARCH_VERSION:-opensearch}.yml:ro - - ../services/elasticsearch/config/log4j2_${ELASTICSEARCH_VERSION:-opensearch}.properties:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/config/log4j2.properties:ro + - ../services/elasticsearch/search-entrypoint.sh:/usr/local/bin/cogstack-search-entrypoint.sh:ro + - ../services/elasticsearch/config/${ELASTICSEARCH_VERSION:-opensearch}.yml:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/config-input/${ELASTICSEARCH_VERSION:-opensearch}.yml:ro + - ../services/elasticsearch/config/log4j2_${ELASTICSEARCH_VERSION:-opensearch}.properties:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/config-input/log4j2.properties:ro + # Supply Performance Analyzer files missing from some OpenSearch images. + - ../services/elasticsearch/config/performance-analyzer/performance-analyzer.properties:/usr/share/opensearch/config-input/opensearch-performance-analyzer/performance-analyzer.properties:ro + - ../services/elasticsearch/config/performance-analyzer/plugin-stats-metadata:/usr/share/opensearch/config-input/opensearch-performance-analyzer/plugin-stats-metadata:ro + - ../services/elasticsearch/config/performance-analyzer/agent-stats-metadata:/usr/share/opensearch/config-input/opensearch-performance-analyzer/agent-stats-metadata:ro # Shared root CA + admin certs - - ../security/certificates/elastic/${ELASTICSEARCH_VERSION:-opensearch}/elastic-stack-ca.crt.pem:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/config/root-ca.crt:ro + - ../security/certificates/elastic/${ELASTICSEARCH_VERSION:-opensearch}/elastic-stack-ca.crt.pem:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/config-input/root-ca.crt:ro # OPENSEARCH specific (always mounted even if unused) - - ../security/certificates/elastic/opensearch/admin.crt:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/config/admin.crt:ro - - ../security/certificates/elastic/opensearch/admin.key.pem:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/config/admin.key.pem:ro + - ../security/certificates/elastic/opensearch/admin.crt:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/config-input/admin.crt:ro + - ../security/certificates/elastic/opensearch/admin.key.pem:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/config-input/admin.key.pem:ro # Shared roles - - ../security/es_roles/elasticsearch/role_mapping.yml:/usr/share/elasticsearch/config/role_mapping.yml:ro - - ../security/es_roles/elasticsearch/roles.yml:/usr/share/elasticsearch/config/roles.yml:ro - - ../security/es_roles/opensearch/config.yml:/usr/share/opensearch/config/opensearch-security/config.yml:ro - - ../security/es_roles/opensearch/internal_users.yml:/usr/share/opensearch/config/opensearch-security/internal_users.yml:ro - - ../security/es_roles/opensearch/roles_mapping.yml:/usr/share/opensearch/config/opensearch-security/roles_mapping.yml:ro - - ../security/es_roles/opensearch/roles.yml:/usr/share/opensearch/config/opensearch-security/roles.yml:ro + - ../security/es_roles/elasticsearch/role_mapping.yml:/usr/share/elasticsearch/config-input/role_mapping.yml:ro + - ../security/es_roles/elasticsearch/roles.yml:/usr/share/elasticsearch/config-input/roles.yml:ro + - ../security/es_roles/opensearch/config.yml:/usr/share/opensearch/config-input/opensearch-security/config.yml:ro + - ../security/es_roles/opensearch/internal_users.yml:/usr/share/opensearch/config-input/opensearch-security/internal_users.yml:ro + - ../security/es_roles/opensearch/roles_mapping.yml:/usr/share/opensearch/config-input/opensearch-security/roles_mapping.yml:ro + - ../security/es_roles/opensearch/roles.yml:/usr/share/opensearch/config-input/opensearch-security/roles.yml:ro # Shared local ML model repository. Register files with a # file:///usr/share/opensearch/models/.zip URL. - ../services/elasticsearch/models:/usr/share/opensearch/models:ro @@ -154,6 +159,9 @@ x-es-common-volumes: &es-common-volumes x-es-common: &es-common <<: *common-ulimits image: ${ELASTICSEARCH_DOCKER_IMAGE:-opensearchproject/opensearch:latest} + entrypoint: ["/bin/bash", "/usr/local/bin/cogstack-search-entrypoint.sh"] + environment: + ELASTICSEARCH_VERSION: ${ELASTICSEARCH_VERSION:-opensearch} shm_size: ${ELASTICSEARCH_DOCKER_SHM_SIZE:-1g} restart: unless-stopped env_file: *es-env @@ -339,6 +347,21 @@ services: #---------------------------------------------------------------------------# # ElasticSearch cluster # #---------------------------------------------------------------------------# + opensearch-performance-analyzer-init: + # Named volumes previously mounted over metadata files are owned by root. + # Prepare the database directories for OpenSearch's uid/gid without touching data. + image: ${ELASTICSEARCH_DOCKER_IMAGE:-opensearchproject/opensearch:latest} + user: "0" + entrypoint: ["/bin/sh", "-ec"] + command: + - chown 1000:1000 /pa-1 /pa-2 /pa-3 + restart: "no" + network_mode: none + volumes: + - ${ELASTICSEARCH_NODE_1_PA_VOL_NAME:-elasticsearch-vol-performance-analyzer-1}:/pa-1 + - ${ELASTICSEARCH_NODE_2_PA_VOL_NAME:-elasticsearch-vol-performance-analyzer-2}:/pa-2 + - ${ELASTICSEARCH_NODE_3_PA_VOL_NAME:-elasticsearch-vol-performance-analyzer-3}:/pa-3 + es_native_create_certs: container_name: es_create_certs image: docker.elastic.co/elasticsearch/elasticsearch:latest @@ -358,6 +381,9 @@ services: elasticsearch-base: <<: *es-common profiles: ["_elasticsearch_base"] + depends_on: + opensearch-performance-analyzer-init: + condition: service_completed_successfully security_opt: - seccomp=unconfined volumes: *es-common-volumes @@ -369,15 +395,15 @@ services: environment: node.name: ${ELASTICSEARCH_NODE_1_NAME:-es01} volumes: - - ../security/certificates/elastic/${ELASTICSEARCH_VERSION:-opensearch}/elasticsearch/${ES_INSTANCE_NAME_1:-elasticsearch-1}/${ES_INSTANCE_NAME_1:-elasticsearch-1}.p12:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/config/esnode.p12:ro - - ../security/certificates/elastic/${ELASTICSEARCH_VERSION:-opensearch}/elasticsearch/${ES_INSTANCE_NAME_1:-elasticsearch-1}/${ES_INSTANCE_NAME_1:-elasticsearch-1}.crt:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/config/esnode.crt:ro - - ../security/certificates/elastic/${ELASTICSEARCH_VERSION:-opensearch}/elasticsearch/${ES_INSTANCE_NAME_1:-elasticsearch-1}/${ES_INSTANCE_NAME_1:-elasticsearch-1}.key:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/config/esnode.key:ro + - ../security/certificates/elastic/${ELASTICSEARCH_VERSION:-opensearch}/elasticsearch/${ES_INSTANCE_NAME_1:-elasticsearch-1}/${ES_INSTANCE_NAME_1:-elasticsearch-1}.p12:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/config-input/esnode.p12:ro + - ../security/certificates/elastic/${ELASTICSEARCH_VERSION:-opensearch}/elasticsearch/${ES_INSTANCE_NAME_1:-elasticsearch-1}/${ES_INSTANCE_NAME_1:-elasticsearch-1}.crt:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/config-input/esnode.crt:ro + - ../security/certificates/elastic/${ELASTICSEARCH_VERSION:-opensearch}/elasticsearch/${ES_INSTANCE_NAME_1:-elasticsearch-1}/${ES_INSTANCE_NAME_1:-elasticsearch-1}.key:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/config-input/esnode.key:ro # ES data persistence - ${ELASTICSEARCH_NODE_1_DATA_VOL_NAME:-elasticsearch-vol-1}:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/data # ES logs - ${ELASTICSEARCH_NODE_1_LOG_VOL_NAME:-elasticsearch-vol-log-1}:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/logs - # OpenSearch performance analyzer - - ${ELASTICSEARCH_NODE_1_PA_VOL_NAME:-elasticsearch-vol-performance-analyzer-1}:/usr/share/opensearch/config/opensearch-performance-analyzer/plugin-stats-metadata:rw + # Performance Analyzer metrics database; metadata above is a regular config file. + - ${ELASTICSEARCH_NODE_1_PA_VOL_NAME:-elasticsearch-vol-performance-analyzer-1}:/var/lib/opensearch-performance-analyzer:rw ports: - "${ELASTICSEARCH_NODE_1_OUTPUT_PORT:-9200}:9200" - "${ELASTICSEARCH_NODE_1_COMM_OUTPUT_PORT:-9300}:9300" @@ -390,15 +416,15 @@ services: environment: node.name: ${ELASTICSEARCH_NODE_2_NAME:-es02} volumes: - - ../security/certificates/elastic/${ELASTICSEARCH_VERSION:-opensearch}/elasticsearch/${ES_INSTANCE_NAME_2:-elasticsearch-2}/${ES_INSTANCE_NAME_2:-elasticsearch-2}.p12:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/config/esnode.p12:ro - - ../security/certificates/elastic/${ELASTICSEARCH_VERSION:-opensearch}/elasticsearch/${ES_INSTANCE_NAME_2:-elasticsearch-2}/${ES_INSTANCE_NAME_2:-elasticsearch-2}.crt:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/config/esnode.crt:ro - - ../security/certificates/elastic/${ELASTICSEARCH_VERSION:-opensearch}/elasticsearch/${ES_INSTANCE_NAME_2:-elasticsearch-2}/${ES_INSTANCE_NAME_2:-elasticsearch-2}.key:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/config/esnode.key:ro + - ../security/certificates/elastic/${ELASTICSEARCH_VERSION:-opensearch}/elasticsearch/${ES_INSTANCE_NAME_2:-elasticsearch-2}/${ES_INSTANCE_NAME_2:-elasticsearch-2}.p12:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/config-input/esnode.p12:ro + - ../security/certificates/elastic/${ELASTICSEARCH_VERSION:-opensearch}/elasticsearch/${ES_INSTANCE_NAME_2:-elasticsearch-2}/${ES_INSTANCE_NAME_2:-elasticsearch-2}.crt:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/config-input/esnode.crt:ro + - ../security/certificates/elastic/${ELASTICSEARCH_VERSION:-opensearch}/elasticsearch/${ES_INSTANCE_NAME_2:-elasticsearch-2}/${ES_INSTANCE_NAME_2:-elasticsearch-2}.key:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/config-input/esnode.key:ro # ES data persistence - ${ELASTICSEARCH_NODE_2_DATA_VOL_NAME:-elasticsearch-vol-2}:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/data # ES logs - ${ELASTICSEARCH_NODE_2_LOG_VOL_NAME:-elasticsearch-vol-log-2}:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/logs - # OpenSearch performance analyzer - - ${ELASTICSEARCH_NODE_2_PA_VOL_NAME:-elasticsearch-vol-performance-analyzer-2}:/usr/share/opensearch/config/opensearch-performance-analyzer/plugins-stats-metadata:rw + # Performance Analyzer metrics database; metadata above is a regular config file. + - ${ELASTICSEARCH_NODE_2_PA_VOL_NAME:-elasticsearch-vol-performance-analyzer-2}:/var/lib/opensearch-performance-analyzer:rw ports: - "${ELASTICSEARCH_NODE_2_OUTPUT_PORT:-9201}:9200" - "${ELASTICSEARCH_NODE_2_COMM_OUTPUT_PORT:-9301}:9300" @@ -411,15 +437,15 @@ services: environment: node.name: ${ELASTICSEARCH_NODE_3_NAME:-es03} volumes: - - ../security/certificates/elastic/${ELASTICSEARCH_VERSION:-opensearch}/elasticsearch/${ES_INSTANCE_NAME_3:-elasticsearch-3}/${ES_INSTANCE_NAME_3:-elasticsearch-3}.p12:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/config/esnode.p12:ro - - ../security/certificates/elastic/${ELASTICSEARCH_VERSION:-opensearch}/elasticsearch/${ES_INSTANCE_NAME_3:-elasticsearch-3}/${ES_INSTANCE_NAME_3:-elasticsearch-3}.crt:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/config/esnode.crt:ro - - ../security/certificates/elastic/${ELASTICSEARCH_VERSION:-opensearch}/elasticsearch/${ES_INSTANCE_NAME_3:-elasticsearch-3}/${ES_INSTANCE_NAME_3:-elasticsearch-3}.key:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/config/esnode.key:ro + - ../security/certificates/elastic/${ELASTICSEARCH_VERSION:-opensearch}/elasticsearch/${ES_INSTANCE_NAME_3:-elasticsearch-3}/${ES_INSTANCE_NAME_3:-elasticsearch-3}.p12:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/config-input/esnode.p12:ro + - ../security/certificates/elastic/${ELASTICSEARCH_VERSION:-opensearch}/elasticsearch/${ES_INSTANCE_NAME_3:-elasticsearch-3}/${ES_INSTANCE_NAME_3:-elasticsearch-3}.crt:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/config-input/esnode.crt:ro + - ../security/certificates/elastic/${ELASTICSEARCH_VERSION:-opensearch}/elasticsearch/${ES_INSTANCE_NAME_3:-elasticsearch-3}/${ES_INSTANCE_NAME_3:-elasticsearch-3}.key:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/config-input/esnode.key:ro # ES data persistence - ${ELASTICSEARCH_NODE_3_DATA_VOL_NAME:-elasticsearch-vol-3}:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/data # ES logs - ${ELASTICSEARCH_NODE_3_LOG_VOL_NAME:-elasticsearch-vol-log-3}:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/logs - # OpenSearch performance analyzer - - ${ELASTICSEARCH_NODE_3_PA_VOL_NAME:-elasticsearch-vol-performance-analyzer-3}:/usr/share/opensearch/config/opensearch-performance-analyzer/plugins-stats-metadata:rw + # Performance Analyzer metrics database; metadata above is a regular config file. + - ${ELASTICSEARCH_NODE_3_PA_VOL_NAME:-elasticsearch-vol-performance-analyzer-3}:/var/lib/opensearch-performance-analyzer:rw ports: - "${ELASTICSEARCH_NODE_3_OUTPUT_PORT:-9202}:9200" - "${ELASTICSEARCH_NODE_3_COMM_OUTPUT_PORT:-9302}:9300" diff --git a/docs/security/elasticsearch_opensearch.md b/docs/security/elasticsearch_opensearch.md index 5db438509..aa28dc3c6 100644 --- a/docs/security/elasticsearch_opensearch.md +++ b/docs/security/elasticsearch_opensearch.md @@ -44,6 +44,60 @@ ELASTICSEARCH_VERSION=elasticsearch This ensures the correct certificate directory (`elasticsearch` or `opensearch`) is mounted into containers. +#### OpenSearch security setting deprecations + +The shared `services/elasticsearch/config/opensearch.yml` is used by Docker Compose and the OpenSearch Helm chart. It uses `transport.ssl.enforce_hostname_verification` instead of the deprecated `plugins.security.ssl.transport.enforce_hostname_verification`. The value remains `false`, preserving the existing transport TLS behavior. Enable verification only after ensuring node certificates cover the hostnames used for inter-node connections. See [Configuring TLS certificates](https://docs.opensearch.org/latest/security/configuration/tls/#advanced-hostname-verification-and-dns-lookup). + +The deprecated `plugins.security.system_indices.indices` list has no replacement YAML setting. The bundled plugins register their system index descriptors with OpenSearch. The shared configuration keeps `plugins.security.system_indices.enabled: true` and `plugins.security.system_indices.permission.enabled: true`, so registered system indices remain protected and roles can still grant explicit `system:admin/system_index` access. See [System indexes](https://docs.opensearch.org/latest/security/configuration/system-indices/). + +Before applying this configuration to an existing cluster, check for legacy or custom indices that depended on the old list. Plugin registrations do not necessarily cover every name matched by the old wildcards; for example, `.opendistro-notebooks` is a legacy name, while the current Observability plugin registers `.opensearch-notebooks`. Keep any required legacy protection until the owning plugin registers those indices or the data has been migrated. Such deployments may need to retain the deprecated list temporarily. + +Apply the same configuration to every OpenSearch node and perform a rolling restart, waiting for each node to rejoin and the cluster to recover before restarting the next. These are static node settings; updating the Security plugin roles or calling its configuration API does not apply them. For Helm deployments, upgrade the ConfigMap and restart the OpenSearch pods; the chart mounts `opensearch.yml` using `subPath`. + +#### Field masking salt and datasource encryption key + +The shared OpenSearch configuration reads `plugins.security.compliance.salt` from `OPENSEARCH_COMPLIANCE_SALT` and `plugins.query.datasources.encryption.masterkey` from `OPENSEARCH_DATASOURCE_MASTER_KEY`. All nodes must receive identical values. Compose reads them from `security/env/users_elasticsearch.env`; the Helm chart imports them into the credentials Secret and references that Secret from every OpenSearch pod. + +The checked-in values are public development defaults. Supply deployment-specific values through local configuration or an external secret store for production. Generate a 16-character ASCII salt with `openssl rand -hex 8` and a 32-character datasource key with `openssl rand -hex 16`. Preserve both values across restarts and keep a backup of the key: changing it prevents decryption of credentials already stored in SQL/PPL datasource definitions. Changing the salt changes field-masking hashes. See [Field masking](https://docs.opensearch.org/latest/security/access-control/field-masking/#set-the-salt-setting) and [Data sources](https://docs.opensearch.org/latest/dashboards/management/data-sources/). + +For Helm, supply these keys in `usersEnvFile.raw` together with the existing credential keys, or include them in `credentials.existingSecret`. When overriding `usersEnvFile.raw` without these keys, set `credentials.complianceSalt` and `credentials.datasourceMasterKey` explicitly. Recreate Compose containers one node at a time after changing their environment; `docker compose restart` does not reload `env_file` values. + +OpenSearch Security 3.9.0 can still log one warning containing its built-in salt during startup. Its internal `FieldMasking.Config.DEFAULT` is initialized from `Settings.EMPTY`, which creates a default `Salt` before the node-specific configuration is used. Inspection of the installed 3.9.0.0 plugin confirms this initialization path. A single such warning does not establish that field masking uses the default salt. Keep the configured salt consistent across nodes. + +#### Runtime configuration permissions and JVM warnings + +The shared `services/elasticsearch/search-entrypoint.sh` copies read-only deployment inputs from `config-input/` into the image's writable `config/` directory before calling the original image entrypoint. Image defaults, including JVM configuration and the keystore, are preserved. For OpenSearch, runtime configuration directories are set to `0700` and files to `0600`; host source permissions are left unchanged. Compose and Helm both use this startup hook. Configuration and certificate changes are copied on container startup, so restart or recreate the container after changing inputs. + +The startup hook sets `DISABLE_INSTALL_DEMO_CONFIG=true` because the deployment supplies its own security configuration and certificates. This skips the demo installer while retaining the Security plugin. The shared JVM options include `--enable-native-access=ALL-UNNAMED` for dependencies that load native libraries. See [Docker configuration](https://docs.opensearch.org/latest/install-and-configure/install-opensearch/docker/) and [JDK native access](https://bugs.openjdk.org/browse/JDK-8334569). + +Messages about Byte Buddy's `sun.misc.Unsafe` calls and missing SLF4J providers originate in bundled dependencies. Resolving them requires compatible upstream dependency changes. Optional-plugin and incubating-vector messages can also appear during normal startup; check node readiness and cluster health to establish whether startup completed. + +#### Repeated startup and out-of-memory exits + +The development configuration allows 2 GiB per OpenSearch node while retaining a 512 MiB Java heap. OpenSearch 3.9 plugin initialization exceeded the former 1 GiB container limit in this deployment, causing repeated Docker `oom` events followed by exit code `137`. The container limit must also cover direct buffers, native allocations, thread stacks, and plugin memory. Ensure the Docker VM or Kubernetes worker has enough memory for every node and the other services. + +Check Docker's event history when startup repeats without a Java exception. `OOMKilled` and cgroup memory counters describe the current container execution and can reset after an automatic restart. Historical `oom` and `die` events identify the cause of earlier executions. See [Docker resource constraints](https://docs.docker.com/engine/containers/resource_constraints/). + +#### Performance Analyzer configuration + +`services/elasticsearch/config/performance-analyzer/` supplies `performance-analyzer.properties`, `plugin-stats-metadata`, and `agent-stats-metadata` for both Compose and Helm. This also covers OpenSearch images that omit these configuration files. See [Performance Analyzer configuration](https://docs.opensearch.org/latest/monitoring-plugins/pa/index/#configure-performance-analyzer). + +The OpenSearch 3.9 image also contains a legacy agent launcher whose `PerformanceAnalyzerApp` class is absent because the RCA application was removed. This is separate from the in-process collectors loading their configuration successfully. The shared entrypoint detects missing RCA library JARs, reports that the agent and port 9600 API are unavailable, and sets `DISABLE_PERFORMANCE_ANALYZER_AGENT_CLI=true`. An explicitly supplied flag takes precedence for custom images. The collector plugin remains installed. See [the upstream missing-RCA issue](https://github.com/opensearch-project/performance-analyzer/issues/904). + +The metadata paths must be regular files. The former volume mounts at `plugin-stats-metadata` and `plugins-stats-metadata` created directories at those paths. Performance Analyzer volumes now hold metrics databases at `/var/lib/opensearch-performance-analyzer`, while temporary metrics stay in `/dev/shm/performanceanalyzer/`. Compose runs a small initialization service to assign the database volume roots to OpenSearch's UID/GID 1000; Helm uses the pod's `fsGroup` for volume access. + +Recreate existing Compose containers to apply the corrected mounts, waiting for each node to recover before moving to the next. Load the deployment environment first, as for other Compose commands. For example: + +```bash +source deploy/export_env_vars.sh +docker compose -f deploy/services.yml up -d elasticsearch-1 +# Wait for node 1 to rejoin and the cluster to recover. +docker compose -f deploy/services.yml up -d elasticsearch-2 +# Repeat for elasticsearch-3 if it is deployed. +``` + +Keep the existing data and log volumes. No volume deletion is needed for this repair. + --- ### 🧩 Common certificate layout diff --git a/security/env/users_elasticsearch.env b/security/env/users_elasticsearch.env index e6319d23d..387b1d298 100644 --- a/security/env/users_elasticsearch.env +++ b/security/env/users_elasticsearch.env @@ -8,6 +8,13 @@ ELASTIC_USER=elastic ELASTIC_PASSWORD=admin OPENSEARCH_INITIAL_ADMIN_PASSWORD=Kibanaserver!1 +# Public development values, identical on all OpenSearch nodes. +# Replace through deployment-local configuration or an external secret store. +# Keep the salt stable (16 ASCII characters) and preserve the datasource key +# (16, 24, or 32 ASCII characters) to decrypt existing datasource credentials. +OPENSEARCH_COMPLIANCE_SALT= +OPENSEARCH_DATASOURCE_MASTER_KEY= + # specify here the kibana user and password that # will be used by kibana when connecting to ES KIBANA_USER=kibanaserver diff --git a/services/elasticsearch/config/opensearch.yml b/services/elasticsearch/config/opensearch.yml index f996d13e0..80cb72eb0 100644 --- a/services/elasticsearch/config/opensearch.yml +++ b/services/elasticsearch/config/opensearch.yml @@ -15,7 +15,7 @@ discovery.type: zen plugins.security.ssl.transport.pemcert_filepath: esnode.crt plugins.security.ssl.transport.pemkey_filepath: esnode.key plugins.security.ssl.transport.pemtrustedcas_filepath: root-ca.crt -plugins.security.ssl.transport.enforce_hostname_verification: false +transport.ssl.enforce_hostname_verification: false #plugins.security.ssl.transport.keystore_filepath: esnode-pcks12.key #plugins.security.ssl.transport.truststore_filepath: esnode-pcks12.key plugins.security.ssl.http.enabled: true @@ -59,6 +59,11 @@ plugins.security.ssl.http.clientauth_mode : OPTIONAL plugins.security.roles_mapping_resolution: MAPPING_ONLY +# Keep these values identical on every node. Shared development values are in +# security/env/users_elasticsearch.env; inject deployment-specific values in production. +plugins.security.compliance.salt: ${OPENSEARCH_COMPLIANCE_SALT} +plugins.query.datasources.encryption.masterkey: ${OPENSEARCH_DATASOURCE_MASTER_KEY} + ############## REST Management API configuration settings ############## # Enable or disable role based access to the REST management API # Default is that no role is allowed to access the REST management API. @@ -176,9 +181,9 @@ plugins.security.audit.config.enable_ssl_client_auth: true # Specify a list of roles a user must be member of to touch any protected index. # plugins.security.protected_indices.roles: ['all_access'] -# System indices are similar to security index, except the contents are not encrypted. -# Indices configured as system indices can be accessed by only super-admin and no role will provide access to these indices. -# Enable system indices +# Bundled plugins register their system indices with OpenSearch. Protect those +# registered indices without the deprecated plugins.security.system_indices.indices list. +# Allow explicit system:admin/system_index role permissions as well as super-admin access. plugins.security.system_indices.enabled: true plugins.security.system_indices.permission.enabled: true @@ -229,22 +234,6 @@ plugins.calcite.enabled: true # WARNING: exists only in OpenSearch > 3.4, please DISABLE if you are not on this version as this setting doesn't exist. plugins.calcite.all_join_types.allowed: true -# Specify a list of indices to mark as system. These indices will only be visible / mutable by members of the above setting, in addition to needing permission to the index via a normal role. -plugins.security.system_indices.indices: [ - ".opensearch_dashboards*", - ".opensearch-alerting-config", - ".opensearch-notifications-*", - ".opendistro-ism-*", - ".opendistro-reports-*", - ".opendistro-asynchronous-search-response*", - ".replication-metadata-store", - ".opendistro-notebooks", - ".opendistro-anomaly-results*", - ".opendistro-anomaly-detector*", - ".opendistro-anomaly-checkpoints", - ".opendistro-anomaly-detection-state" -] - # coordinator-level kill switch for searches search.cancel_after_time_interval: 5m diff --git a/services/elasticsearch/config/performance-analyzer/agent-stats-metadata b/services/elasticsearch/config/performance-analyzer/agent-stats-metadata new file mode 100644 index 000000000..c358dc469 --- /dev/null +++ b/services/elasticsearch/config/performance-analyzer/agent-stats-metadata @@ -0,0 +1 @@ +Program=PerformanceAnalyzerAgent diff --git a/services/elasticsearch/config/performance-analyzer/performance-analyzer.properties b/services/elasticsearch/config/performance-analyzer/performance-analyzer.properties new file mode 100644 index 000000000..1db3d637a --- /dev/null +++ b/services/elasticsearch/config/performance-analyzer/performance-analyzer.properties @@ -0,0 +1,9 @@ +# Shared Performance Analyzer configuration for Docker Compose and Helm. +metrics-location = /dev/shm/performanceanalyzer/ +metrics-deletion-interval = 1 +cleanup-metrics-db-files = true +webservice-listener-port = 9600 +metrics-db-file-prefix-path = /var/lib/opensearch-performance-analyzer/metricsdb_ +https-enabled = false +plugin-stats-metadata = plugin-stats-metadata +agent-stats-metadata = agent-stats-metadata diff --git a/services/elasticsearch/config/performance-analyzer/plugin-stats-metadata b/services/elasticsearch/config/performance-analyzer/plugin-stats-metadata new file mode 100644 index 000000000..8ef2d30c4 --- /dev/null +++ b/services/elasticsearch/config/performance-analyzer/plugin-stats-metadata @@ -0,0 +1 @@ +Program=PerformanceAnalyzerPlugin diff --git a/services/elasticsearch/search-entrypoint.sh b/services/elasticsearch/search-entrypoint.sh new file mode 100644 index 000000000..a50e0990d --- /dev/null +++ b/services/elasticsearch/search-entrypoint.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Keep host bind mounts and Kubernetes projections separate from the writable +# runtime configuration. The image's bundled config (including jvm.options and +# its keystore) remains in place; deployment files override only supplied files. +backend="${ELASTICSEARCH_VERSION:-opensearch}" +case "$backend" in + opensearch|elasticsearch) ;; + *) echo "Unsupported search backend: $backend" >&2; exit 1 ;; +esac + +search_home="${COGSTACK_SEARCH_HOME:-/usr/share/$backend}" +input_dir="$search_home/config-input" +config_dir="$search_home/config" +if [[ ! -f "$input_dir/$backend.yml" ]]; then + echo "Missing deployment configuration: $input_dir/$backend.yml" >&2 + exit 1 +fi + +umask 077 +cp -RL "$input_dir/." "$config_dir/" + +if [[ "$backend" == opensearch ]]; then + # find is not installed in all official images. Use Bash to normalize modes + # without requiring root or changing the source files on the host. + shopt -s nullglob dotglob + secure_config_directory() { + local directory="$1" item + chmod 0700 "$directory" + for item in "$directory"/*; do + if [[ -L "$item" ]]; then + echo "Runtime configuration must contain regular files, not symlinks: $item" >&2 + exit 1 + elif [[ -d "$item" ]]; then + secure_config_directory "$item" + else + chmod 0600 "$item" + fi + done + } + secure_config_directory "$config_dir" + export DISABLE_INSTALL_DEMO_CONFIG=true + # OpenSearch 3.x removed the RCA application, but some images still contain + # its agent launcher. Avoid starting a JVM whose main class is not installed. + # An explicitly supplied flag takes precedence for custom images. + if [[ -z "${DISABLE_PERFORMANCE_ANALYZER_AGENT_CLI+x}" ]] \ + && ! compgen -G "$search_home/performance-analyzer-rca/lib/*.jar" >/dev/null; then + export DISABLE_PERFORMANCE_ANALYZER_AGENT_CLI=true + echo "Performance Analyzer RCA libraries are absent; the legacy agent and port 9600 API are unavailable." + fi + exec "$search_home/opensearch-docker-entrypoint.sh" "$@" +else + exec /usr/local/bin/docker-entrypoint.sh "$@" +fi From c26b8724907b0ded6461c1c91abd393c1491972d Mon Sep 17 00:00:00 2001 From: vladd-bit Date: Fri, 23 Oct 2026 17:14:17 +0100 Subject: [PATCH 07/28] Deploy: fix script perms. --- services/elasticsearch/search-entrypoint.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 services/elasticsearch/search-entrypoint.sh diff --git a/services/elasticsearch/search-entrypoint.sh b/services/elasticsearch/search-entrypoint.sh old mode 100644 new mode 100755 From 40c42142ac3bc04bd5cba6cea92380f0d10db7d6 Mon Sep 17 00:00:00 2001 From: vladd-bit Date: Wed, 28 Oct 2026 08:45:41 +0000 Subject: [PATCH 08/28] Scripts: nifi smoke test updates. --- scripts/tests/smoke_nifi_services.sh | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/scripts/tests/smoke_nifi_services.sh b/scripts/tests/smoke_nifi_services.sh index 9dca31615..e105a7390 100755 --- a/scripts/tests/smoke_nifi_services.sh +++ b/scripts/tests/smoke_nifi_services.sh @@ -20,7 +20,8 @@ fi # shellcheck disable=SC1090 source "$SMOKE_HELPERS" -HOST="${NIFI_SMOKE_HOST:-localhost}" +# Match nginx's canonical loopback host; localhost redirects to 127.0.0.1. +HOST="${NIFI_SMOKE_HOST:-127.0.0.1}" NIFI_PORT="${NIFI_EXTERNAL_PORT_NGINX:-8443}" # The sourced smoke helper reads this shared variable when checks run. # shellcheck disable=SC2034 @@ -37,24 +38,34 @@ SMOKE_CHECKS=( check_authenticated_nifi_api() { local token + local token_response local code - if ! token="$( + if ! token_response="$( curl \ --silent \ --show-error \ - --fail \ --cacert "$SMOKE_CA_CERT" \ --connect-timeout "$SMOKE_CONNECT_TIMEOUT" \ --max-time "$SMOKE_MAX_TIME" \ --data-urlencode "username=${NIFI_SINGLE_USER_CREDENTIALS_USERNAME}" \ --data-urlencode "password=${NIFI_SINGLE_USER_CREDENTIALS_PASSWORD}" \ + --write-out $'\n%{http_code}' \ "${NIFI_BASE_URL}/nifi-api/access/token" )"; then echo "FAIL: nifi-api - authentication failed" >&2 return 1 fi + # NiFi creates tokens with HTTP 201. In particular, curl considers redirects + # successful, but their HTML response bodies must never become bearer tokens. + code="${token_response##*$'\n'}" + if [[ "$code" != "201" ]]; then + echo "FAIL: nifi-api - authentication returned HTTP ${code} (expected HTTP 201)" >&2 + return 1 + fi + token="${token_response%$'\n'*}" + if [[ -z "$token" ]]; then echo "FAIL: nifi-api - authentication returned an empty token" >&2 return 1 From cc1e81eddb2e59b0457579b7101874a03defaa3c Mon Sep 17 00:00:00 2001 From: vladd-bit Date: Thu, 29 Oct 2026 08:04:26 +0000 Subject: [PATCH 09/28] Deploy/services: refined OpenSearch integration with AI agents... added support for local farmeworks (sglang) --- .gitignore | 1 + deploy/Makefile | 14 +- deploy/export_env_vars.sh | 1 + deploy/nifi.env | 4 +- deploy/sglang.env | 18 ++ docs/deploy/services.md | 7 + docs/deploy/sglang.md | 76 ++++++ mkdocs.yml | 1 + scripts/opensearch-ai-init.sh | 290 +++++++++++++++-------- scripts/tests/test_opensearch_ai_init.py | 235 ++++++++++++++++++ services/kibana/config/opensearch.yml | 2 + 11 files changed, 547 insertions(+), 102 deletions(-) create mode 100644 deploy/sglang.env create mode 100644 docs/deploy/sglang.md create mode 100644 scripts/tests/test_opensearch_ai_init.py diff --git a/.gitignore b/.gitignore index f5fb47b92..5cb535b82 100644 --- a/.gitignore +++ b/.gitignore @@ -53,4 +53,5 @@ site/ !*.env.template !**/*.env.template !deploy/ollama.env +!deploy/sglang.env !deploy/openwebui.env diff --git a/deploy/Makefile b/deploy/Makefile index 31a0a818f..a7a5ba5c9 100644 --- a/deploy/Makefile +++ b/deploy/Makefile @@ -363,8 +363,16 @@ pull-ollama-model: start-ollama ## Pull the model configured by OLLAMA_MODEL setup-ollama: pull-ollama-model ## Start Ollama and pull its configured model -opensearch-ai-init: start-ollama-openai-proxy ## Create or update all Ollama-backed OpenSearch Dashboards AI agents and configs - $(WITH_ENV) ../scripts/opensearch-ai-init.sh +opensearch-ai-init: ## Create or update OpenSearch AI connectors and agents using OPENSEARCH_AI_PROVIDER + $(WITH_ENV) \ + case "$${OPENSEARCH_AI_PROVIDER:-ollama}" in \ + ollama) docker compose -f services.yml $(DC_START_CMD) --build ollama-openai-proxy ;; \ + sglang) ;; \ + *) echo "Unsupported OPENSEARCH_AI_PROVIDER: $${OPENSEARCH_AI_PROVIDER}" >&2; exit 1 ;; \ + esac && ../scripts/opensearch-ai-init.sh + +opensearch-ai-init-sglang: ## Connect all OpenSearch AI features to the locally hosted SGLang server + $(WITH_ENV) OPENSEARCH_AI_PROVIDER=sglang ../scripts/opensearch-ai-init.sh start-openwebui: ## Start Open WebUI and its Ollama dependency $(WITH_ENV) docker compose -f services.yml $(DC_START_CMD) open-webui @@ -415,7 +423,7 @@ start-data-infra: start-nifi start-elastic start-samples ## Start core data infr start-all: start-data-infra start-medcat-service start-ocr-services ## Start all major services -.PHONY: start-all start-data-infra start-nifi start-nifi-dev start-nifi-dev-build build-nifi-image start-elastic start-ollama start-ollama-openai-proxy pull-ollama-model setup-ollama opensearch-ai-init start-openwebui restart-openwebui status-openwebui logs-openwebui start-samples start-clickhouse status-clickhouse logs-clickhouse start-jupyter +.PHONY: start-all start-data-infra start-nifi start-nifi-dev start-nifi-dev-build build-nifi-image start-elastic start-ollama start-ollama-openai-proxy pull-ollama-model setup-ollama opensearch-ai-init opensearch-ai-init-sglang start-openwebui restart-openwebui status-openwebui logs-openwebui start-samples start-clickhouse status-clickhouse logs-clickhouse start-jupyter # stop services diff --git a/deploy/export_env_vars.sh b/deploy/export_env_vars.sh index 8b18c355e..db59b8662 100755 --- a/deploy/export_env_vars.sh +++ b/deploy/export_env_vars.sh @@ -30,6 +30,7 @@ env_files=( "$DEPLOY_DIR/nifi.env" "$DEPLOY_DIR/elasticsearch.env" "$DEPLOY_DIR/ollama.env" + "$DEPLOY_DIR/sglang.env" "$DEPLOY_DIR/openwebui.env" "$DEPLOY_DIR/database.env" "$DEPLOY_DIR/network_settings.env" diff --git a/deploy/nifi.env b/deploy/nifi.env index 7c383ae08..7afe3520e 100644 --- a/deploy/nifi.env +++ b/deploy/nifi.env @@ -5,10 +5,10 @@ ############################################################################################################################## NIFI_JVM_OPTS="-XX:+UseG1GC -XX:MaxGCPauseMillis=250 -XX:+ParallelRefProcEnabled -Xlog:gc*:file=/opt/nifi/nifi-current/logs/gc.log:time,uptime,level,tags:filecount=5,filesize=50m -Djava.security.egd=file:/dev/./urandom" NIFI_JVM_HEAP_INIT=768m -NIFI_JVM_HEAP_MAX=1g +NIFI_JVM_HEAP_MAX=2g -NIFI_DOCKER_SHM_SIZE=1g +NIFI_DOCKER_SHM_SIZE=512m NIFI_DOCKER_CPU_MIN=1 NIFI_DOCKER_CPU_MAX=1 diff --git a/deploy/sglang.env b/deploy/sglang.env new file mode 100644 index 000000000..f0a976cbe --- /dev/null +++ b/deploy/sglang.env @@ -0,0 +1,18 @@ +####################################################################### SGLang Env vars ####################################################################### + +# OpenSearch AI backend: ollama (default) or sglang. The explicit Make target +# `make opensearch-ai-init-sglang` selects SGLang for that run too. +OPENSEARCH_AI_PROVIDER=${OPENSEARCH_AI_PROVIDER:-sglang} + +# URL as seen from the OpenSearch containers, including /v1. For a server on +# this Docker host use host.docker.internal, not localhost. For another machine +# use its reachable hostname/IP. SGLang must listen on an accessible interface. +SGLANG_BASE_URL=${SGLANG_BASE_URL:-http://host.docker.internal:30000/v1} + +# Exact served model ID from GET /v1/models (or SGLang's --served-model-name). +# Leave empty to discover it automatically when the server exposes one model. +SGLANG_MODEL=${SGLANG_MODEL:-"Qwen3.8-27B-4bit"} + +# Optional: match the server's --api-key. Empty means no Authorization header. +# Prefer exporting the key in your shell instead of saving it in this file. +SGLANG_API_KEY=${SGLANG_API_KEY:-} diff --git a/docs/deploy/services.md b/docs/deploy/services.md index 6c7835467..eaa9ca5f2 100644 --- a/docs/deploy/services.md +++ b/docs/deploy/services.md @@ -466,6 +466,13 @@ Image selection controlled by: --- +## 🤖 OpenSearch AI with SGLang + +To connect OpenSearch Dashboards AI features to an existing SGLang server, see +the [OpenSearch AI with SGLang](./sglang.md) deployment guide. + +--- + ## 🤖 OCR Service High-performance document text-extraction engine for OCR and text processing. diff --git a/docs/deploy/sglang.md b/docs/deploy/sglang.md new file mode 100644 index 000000000..561d092b0 --- /dev/null +++ b/docs/deploy/sglang.md @@ -0,0 +1,76 @@ +# OpenSearch AI with a locally hosted SGLang server + +Configure `deploy/sglang.env` to connect the existing OpenSearch Dashboards AI +features to an already running SGLang server. No SGLang container or Python +environment is created by these commands. + +Run these commands from the repository's `deploy/` directory: + +```bash +# Defaults to http://host.docker.internal:30000/v1 and discovers a single model. +make opensearch-ai-init-sglang + +# Override the URL/model for a different port, host, or served model alias. +SGLANG_BASE_URL=http://host.docker.internal:8000/v1 \ +SGLANG_MODEL=my-served-model \ +make opensearch-ai-init-sglang +``` + +Use the exact model ID returned by `GET /v1/models`, including any alias set by +SGLang's `--served-model-name`. If the server exposes multiple models, specify +`SGLANG_MODEL`. Export `SGLANG_API_KEY` when the server uses `--api-key`; it is +stored as an OpenSearch connector credential and sent as a Bearer token. + +For a server on the Docker host, use `host.docker.internal`, since `localhost` +inside OpenSearch refers to its container. Docker Desktop and Colima provide +this hostname. On Linux Docker Engine, add +`"host.docker.internal:host-gateway"` to the OpenSearch containers' `extra_hosts` +(the `x-common-hosts` anchor in `deploy/services.yml`) and recreate them, or use the +host's reachable IP in `SGLANG_BASE_URL`. For a server on another machine, use +its reachable LAN hostname or IP instead. +Bind SGLang to an interface reachable from Docker, for example `--host 0.0.0.0`. + +Setup checks `/v1/models` from inside OpenSearch before writing configuration. +It adds the exact inference URL to both ML Commons connector allowlists while +preserving existing entries. This uses OpenSearch 3.7+ private connector +settings already enabled in this repository; no OpenSearch restart is needed +for these allowlist updates. + +All six connectors use SGLang: chat, PPL, time-range parsing, Dashboards utility +features, anomaly suggestions, and visualization summaries. Resource names +default to SGLang-specific names. Existing `OPENSEARCH_AI_*_NAME` settings in +`deploy/elasticsearch.env` override those defaults, so pinned Ollama names cause those +same connectors, models, and agents to be updated to use SGLang. Set distinct +resource names if you want both sets to coexist. Setup activates the selected +provider by updating the Dashboards ML configuration documents. Rerunning it +updates existing connectors and agents. + +Choose a chat model and enable the appropriate SGLang tool-call parser for its +model family to use the conversational agent. Anomaly suggestions require +structured JSON output; visualization summaries require a vision model. +SGLang uses its server defaults for reasoning unless +`OPENSEARCH_AI_REASONING_EFFORT` is supplied. Configure model-specific thinking +controls on the server; thinking models may need higher +`OPENSEARCH_AI_MAX_TOKENS` and `OPENSEARCH_AI_UTILITY_MAX_TOKENS` than the current +256/1024 defaults. + +To select SGLang for the generic command, set `OPENSEARCH_AI_PROVIDER=sglang` +in `deploy/sglang.env`, or run: + +```bash +OPENSEARCH_AI_PROVIDER=sglang make opensearch-ai-init +``` + +Ollama remains the default. Switch the active Dashboards features back with: + +```bash +OPENSEARCH_AI_PROVIDER=ollama make opensearch-ai-init +``` + +The generic command starts the Ollama compatibility proxy only when Ollama is +selected. The SGLang path calls its OpenAI-compatible API directly. + +References: [SGLang OpenAI-compatible API](https://docs.sglang.io/docs/basic_usage/openai_api_completions), +[SGLang server arguments](https://docs.sglang.io/docs/advanced_features/server_arguments), +[OpenSearch remote connectors](https://docs.opensearch.org/latest/ml-commons-plugin/remote-models/index/), +[Docker host networking](https://docs.docker.com/compose/how-tos/networking/). diff --git a/mkdocs.yml b/mkdocs.yml index 7d6c4db91..577635861 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -46,6 +46,7 @@ nav: - 📦 Deployment: deploy/deployment.md - ⚙️ Configuration: deploy/configuration.md - 🧩 Services: deploy/services.md + - 🤖 OpenSearch AI with SGLang: deploy/sglang.md - 📛 Troubleshooting: deploy/troubleshooting.md - 🔀 Workflows: - Overview: deploy/workflows.md diff --git a/scripts/opensearch-ai-init.sh b/scripts/opensearch-ai-init.sh index a5c387b5f..32d383490 100755 --- a/scripts/opensearch-ai-init.sh +++ b/scripts/opensearch-ai-init.sh @@ -33,6 +33,10 @@ ENV_OVERRIDE_NAMES=( OLLAMA_CONTAINER_NAME OLLAMA_MODEL OLLAMA_OPENAI_PROXY_CONTAINER_NAME + OPENSEARCH_AI_PROVIDER + SGLANG_BASE_URL + SGLANG_MODEL + SGLANG_API_KEY OPENSEARCH_AI_CHAT_ENDPOINT OPENSEARCH_AI_OPENSEARCH_CONTAINER OPENSEARCH_AI_OPENSEARCH_USER @@ -71,6 +75,7 @@ done load_env_file "$PROJECT_DIR/security/env/users_elasticsearch.env" load_env_file "$PROJECT_DIR/deploy/elasticsearch.env" load_env_file "$PROJECT_DIR/deploy/ollama.env" +load_env_file "$PROJECT_DIR/deploy/sglang.env" for variable_assignment in "${ENV_OVERRIDES[@]}"; do variable_name="${variable_assignment%%=*}" printf -v "$variable_name" '%s' "${variable_assignment#*=}" @@ -82,34 +87,70 @@ OPENSEARCH_USER="${OPENSEARCH_AI_OPENSEARCH_USER:-${ELASTIC_USER:-admin}}" OPENSEARCH_PASSWORD="${OPENSEARCH_AI_OPENSEARCH_PASSWORD:-${ELASTIC_PASSWORD:-}}" OLLAMA_CONTAINER="${OLLAMA_CONTAINER_NAME:-cogstack-ollama}" OLLAMA_OPENAI_PROXY_CONTAINER="${OLLAMA_OPENAI_PROXY_CONTAINER_NAME:-cogstack-ollama-openai-proxy}" -OLLAMA_MODEL_NAME="${OLLAMA_MODEL:-qwen3.5:9b-q4_K_M}" CHAT_ENDPOINT="${OPENSEARCH_AI_CHAT_ENDPOINT:-ollama-openai-proxy:11435}" +PROVIDER="${OPENSEARCH_AI_PROVIDER:-ollama}" +API_KEY="" +CONNECTOR_NAME_PREFIX="Ollama Qwen" +MODEL_NAME_PREFIX="Qwen 3.5" +AGENT_NAME_PREFIX="Qwen" +REMOTE_MODEL_NAME_PREFIX="Qwen 3.5 Ollama" +DEFAULT_CHAT_MODEL_NAME="Qwen 3.5 Ollama Agent Model" +DEFAULT_CHAT_AGENT_NAME="Qwen 3.5 Conversational Agent v2" +DEFAULT_REASONING_EFFORT="none" + +case "$PROVIDER" in + ollama) + PROVIDER_LABEL="Ollama" + MODEL_NAME="${OLLAMA_MODEL:-qwen3.5:9b-q4_K_M}" + BASE_URL="http://ollama:11434/v1" + CHAT_BASE_URL="http://$CHAT_ENDPOINT/v1" + ;; + sglang) + PROVIDER_LABEL="SGLang" + MODEL_NAME="${SGLANG_MODEL:-}" + BASE_URL="${SGLANG_BASE_URL:-http://host.docker.internal:30000/v1}" + BASE_URL="${BASE_URL%/}" + [[ "$BASE_URL" =~ ^https?://[^/?#@[:space:]]+(/[^?#[:space:]]*)?/v1$ && "$BASE_URL" != *\\* ]] \ + || die "SGLANG_BASE_URL must be an http(s) URL ending in /v1, without credentials, query, or fragment" + CHAT_BASE_URL="$BASE_URL" + API_KEY="${SGLANG_API_KEY:-}" + CONNECTOR_NAME_PREFIX="SGLang" + MODEL_NAME_PREFIX="SGLang" + AGENT_NAME_PREFIX="SGLang" + REMOTE_MODEL_NAME_PREFIX="SGLang" + DEFAULT_CHAT_MODEL_NAME="SGLang Agent Model" + DEFAULT_CHAT_AGENT_NAME="SGLang Conversational Agent" + # Reasoning controls vary by model; let SGLang use its server defaults. + DEFAULT_REASONING_EFFORT="" + ;; + *) die "unsupported OPENSEARCH_AI_PROVIDER '$PROVIDER'; expected ollama or sglang" ;; +esac CONNECTION_TIMEOUT="${OPENSEARCH_AI_CONNECTION_TIMEOUT_SECONDS:-120}" READ_TIMEOUT="${OPENSEARCH_AI_READ_TIMEOUT_SECONDS:-360}" DEPLOY_TIMEOUT="${OPENSEARCH_AI_DEPLOY_TIMEOUT_SECONDS:-300}" MAX_TOKENS="${OPENSEARCH_AI_MAX_TOKENS:-256}" UTILITY_MAX_TOKENS="${OPENSEARCH_AI_UTILITY_MAX_TOKENS:-1024}" -REASONING_EFFORT="${OPENSEARCH_AI_REASONING_EFFORT:-none}" +REASONING_EFFORT="${OPENSEARCH_AI_REASONING_EFFORT-$DEFAULT_REASONING_EFFORT}" CHAT_MAX_ITERATIONS="${OPENSEARCH_AI_CHAT_MAX_ITERATIONS:-8}" PPL_RESULT_LIMIT="${OPENSEARCH_AI_PPL_RESULT_LIMIT:-100}" -CHAT_CONNECTOR_NAME="${OPENSEARCH_AI_CHAT_CONNECTOR_NAME:-Ollama Qwen Agent Connector}" -CHAT_MODEL_NAME="${OPENSEARCH_AI_CHAT_MODEL_NAME:-Qwen 3.5 Ollama Agent Model}" -CHAT_AGENT_NAME="${OPENSEARCH_AI_CHAT_AGENT_NAME:-Qwen 3.5 Conversational Agent v2}" -ROOT_AGENT_NAME="${OPENSEARCH_AI_ROOT_AGENT_NAME:-Qwen Dashboards Root Agent}" -PPL_CONNECTOR_NAME="${OPENSEARCH_AI_PPL_CONNECTOR_NAME:-Ollama Qwen PPL Connector}" -PPL_MODEL_NAME="${OPENSEARCH_AI_PPL_MODEL_NAME:-Qwen 3.5 Ollama PPL Model}" -PPL_AGENT_NAME="${OPENSEARCH_AI_PPL_AGENT_NAME:-Qwen PPL Query Assist Agent}" -TIME_RANGE_CONNECTOR_NAME="${OPENSEARCH_AI_TIME_RANGE_CONNECTOR_NAME:-Ollama Qwen Time Range Connector}" -TIME_RANGE_MODEL_NAME="${OPENSEARCH_AI_TIME_RANGE_MODEL_NAME:-Qwen 3.5 Ollama Time Range Model}" -TIME_RANGE_AGENT_NAME="${OPENSEARCH_AI_TIME_RANGE_AGENT_NAME:-Qwen Query Time Range Parser Agent}" -UTILITY_CONNECTOR_NAME="${OPENSEARCH_AI_UTILITY_CONNECTOR_NAME:-Ollama Qwen Dashboards Utility Connector}" -UTILITY_MODEL_NAME="${OPENSEARCH_AI_UTILITY_MODEL_NAME:-Qwen 3.5 Dashboards Utility Model}" -AD_CONNECTOR_NAME="${OPENSEARCH_AI_AD_CONNECTOR_NAME:-Ollama Qwen Anomaly Suggestion Connector}" -AD_MODEL_NAME="${OPENSEARCH_AI_AD_MODEL_NAME:-Qwen 3.5 Anomaly Suggestion Model}" -VISUAL_CONNECTOR_NAME="${OPENSEARCH_AI_VISUAL_CONNECTOR_NAME:-Ollama Qwen Visualization Connector}" -VISUAL_MODEL_NAME="${OPENSEARCH_AI_VISUAL_MODEL_NAME:-Qwen 3.5 Visualization Model}" +CHAT_CONNECTOR_NAME="${OPENSEARCH_AI_CHAT_CONNECTOR_NAME:-$CONNECTOR_NAME_PREFIX Agent Connector}" +CHAT_MODEL_NAME="${OPENSEARCH_AI_CHAT_MODEL_NAME:-$DEFAULT_CHAT_MODEL_NAME}" +CHAT_AGENT_NAME="${OPENSEARCH_AI_CHAT_AGENT_NAME:-$DEFAULT_CHAT_AGENT_NAME}" +ROOT_AGENT_NAME="${OPENSEARCH_AI_ROOT_AGENT_NAME:-$AGENT_NAME_PREFIX Dashboards Root Agent}" +PPL_CONNECTOR_NAME="${OPENSEARCH_AI_PPL_CONNECTOR_NAME:-$CONNECTOR_NAME_PREFIX PPL Connector}" +PPL_MODEL_NAME="${OPENSEARCH_AI_PPL_MODEL_NAME:-$REMOTE_MODEL_NAME_PREFIX PPL Model}" +TIME_RANGE_CONNECTOR_NAME="${OPENSEARCH_AI_TIME_RANGE_CONNECTOR_NAME:-$CONNECTOR_NAME_PREFIX Time Range Connector}" +TIME_RANGE_MODEL_NAME="${OPENSEARCH_AI_TIME_RANGE_MODEL_NAME:-$REMOTE_MODEL_NAME_PREFIX Time Range Model}" +PPL_AGENT_NAME="${OPENSEARCH_AI_PPL_AGENT_NAME:-$AGENT_NAME_PREFIX PPL Query Assist Agent}" +TIME_RANGE_AGENT_NAME="${OPENSEARCH_AI_TIME_RANGE_AGENT_NAME:-$AGENT_NAME_PREFIX Query Time Range Parser Agent}" +UTILITY_CONNECTOR_NAME="${OPENSEARCH_AI_UTILITY_CONNECTOR_NAME:-$CONNECTOR_NAME_PREFIX Dashboards Utility Connector}" +UTILITY_MODEL_NAME="${OPENSEARCH_AI_UTILITY_MODEL_NAME:-$MODEL_NAME_PREFIX Dashboards Utility Model}" +AD_CONNECTOR_NAME="${OPENSEARCH_AI_AD_CONNECTOR_NAME:-$CONNECTOR_NAME_PREFIX Anomaly Suggestion Connector}" +AD_MODEL_NAME="${OPENSEARCH_AI_AD_MODEL_NAME:-$MODEL_NAME_PREFIX Anomaly Suggestion Model}" +VISUAL_CONNECTOR_NAME="${OPENSEARCH_AI_VISUAL_CONNECTOR_NAME:-$CONNECTOR_NAME_PREFIX Visualization Connector}" +VISUAL_MODEL_NAME="${OPENSEARCH_AI_VISUAL_MODEL_NAME:-$MODEL_NAME_PREFIX Visualization Model}" OPENSEARCH_URL="https://localhost:9200" OPENSEARCH_CA="/usr/share/opensearch/config/root-ca.crt" @@ -121,16 +162,23 @@ for command_name in docker jq; do command -v "$command_name" >/dev/null 2>&1 || die "$command_name is required" done +REASONING_OPTION="" +if [[ -n "$REASONING_EFFORT" ]]; then + REASONING_OPTION=", \"reasoning_effort\": $(jq -nc --arg effort "$REASONING_EFFORT" '$effort')" +fi + for integer_value in "$CONNECTION_TIMEOUT" "$READ_TIMEOUT" "$DEPLOY_TIMEOUT" "$MAX_TOKENS" "$UTILITY_MAX_TOKENS" "$CHAT_MAX_ITERATIONS" "$PPL_RESULT_LIMIT"; do [[ "$integer_value" =~ ^[1-9][0-9]*$ ]] || die "timeout and token settings must be positive integers" done [[ "$(docker inspect --format '{{.State.Running}}' "$OPENSEARCH_CONTAINER" 2>/dev/null)" == "true" ]] \ || die "OpenSearch container '$OPENSEARCH_CONTAINER' is not running" -[[ "$(docker inspect --format '{{.State.Running}}' "$OLLAMA_CONTAINER" 2>/dev/null)" == "true" ]] \ - || die "Ollama container '$OLLAMA_CONTAINER' is not running" -[[ "$(docker inspect --format '{{.State.Running}}' "$OLLAMA_OPENAI_PROXY_CONTAINER" 2>/dev/null)" == "true" ]] \ - || die "Ollama OpenAI compatibility proxy '$OLLAMA_OPENAI_PROXY_CONTAINER' is not running" +if [[ "$PROVIDER" == "ollama" ]]; then + [[ "$(docker inspect --format '{{.State.Running}}' "$OLLAMA_CONTAINER" 2>/dev/null)" == "true" ]] \ + || die "Ollama container '$OLLAMA_CONTAINER' is not running" + [[ "$(docker inspect --format '{{.State.Running}}' "$OLLAMA_OPENAI_PROXY_CONTAINER" 2>/dev/null)" == "true" ]] \ + || die "Ollama OpenAI compatibility proxy '$OLLAMA_OPENAI_PROXY_CONTAINER' is not running" +fi os_request() { local method="$1" @@ -181,11 +229,11 @@ wait_for_opensearch() { wait_for_ollama() { local deadline=$((SECONDS + DEPLOY_TIMEOUT)) - log "Waiting for Ollama and ensuring model '$OLLAMA_MODEL_NAME' is available" + log "Waiting for Ollama and ensuring model '$MODEL_NAME' is available" while (( SECONDS < deadline )); do if docker exec "$OLLAMA_CONTAINER" ollama list >/dev/null 2>&1; then - if ! docker exec "$OLLAMA_CONTAINER" ollama show "$OLLAMA_MODEL_NAME" >/dev/null 2>&1; then - docker exec "$OLLAMA_CONTAINER" ollama pull "$OLLAMA_MODEL_NAME" + if ! docker exec "$OLLAMA_CONTAINER" ollama show "$MODEL_NAME" >/dev/null 2>&1; then + docker exec "$OLLAMA_CONTAINER" ollama pull "$MODEL_NAME" fi return 0 fi @@ -194,6 +242,53 @@ wait_for_ollama() { die "Ollama did not become ready within ${DEPLOY_TIMEOUT}s" } +check_sglang() { + local response model_count + local curl_args=(--silent --show-error --fail --connect-timeout "$CONNECTION_TIMEOUT" --max-time "$READ_TIMEOUT") + if [[ -n "$API_KEY" ]]; then + curl_args+=(--header "Authorization: Bearer $API_KEY") + fi + log "Checking SGLang from '$OPENSEARCH_CONTAINER' at $BASE_URL/models" + response="$(docker exec "$OPENSEARCH_CONTAINER" curl "${curl_args[@]}" "$BASE_URL/models")" \ + || die "SGLang is unreachable from OpenSearch; check SGLANG_BASE_URL, API key, listening interface, and host mapping (localhost refers to the container)" + jq -e '.data | type == "array" and length > 0' <<<"$response" >/dev/null \ + || die "SGLang /v1/models did not return a non-empty model list" + if [[ -z "$MODEL_NAME" ]]; then + model_count="$(jq '.data | length' <<<"$response")" + [[ "$model_count" == "1" ]] || die "SGLang exposes multiple models; set SGLANG_MODEL to the desired model ID" + MODEL_NAME="$(jq -r '.data[0].id // empty' <<<"$response")" + fi + if [[ -z "$MODEL_NAME" ]] || ! jq -e --arg model "$MODEL_NAME" 'any(.data[]; .id == $model)' <<<"$response" >/dev/null; then + die "SGLANG_MODEL '$MODEL_NAME' was not found in SGLang /v1/models" + fi + log "Using SGLang model '$MODEL_NAME'" +} + +trust_sglang_endpoint() { + local settings payload + settings="$(os_request GET '/_cluster/settings?include_defaults=true&flat_settings=true')" + # Java Pattern.quote syntax trusts this exact inference URL, including any + # reverse-proxy path. Preserve existing allowlists and transient overrides. + payload="$(jq -c --arg url "$BASE_URL/chat/completions" ' + . as $settings + | ("^\\Q" + $url + "\\E$") as $pattern + | ["plugins.ml_commons.trusted_connector_endpoints_regex", + "plugins.ml_commons.trusted_connector_private_endpoints_regex"] as $keys + | reduce $keys[] as $key ({persistent:{}, transient:{}}; + .persistent[$key] = (($settings.persistent[$key] // $settings.defaults[$key] // []) + [$pattern] | unique) + | if $settings.transient[$key] != null then + .transient[$key] = ($settings.transient[$key] + [$pattern] | unique) + else . end) + | .persistent["plugins.ml_commons.connector.private_ip_enabled"] = true + | if $settings.transient["plugins.ml_commons.connector.private_ip_enabled"] != null then + .transient["plugins.ml_commons.connector.private_ip_enabled"] = true + else . end + | if .transient == {} then del(.transient) else . end + ' <<<"$settings")" + log "Trusting the SGLang inference URL in OpenSearch" + os_request PUT '/_cluster/settings' "$payload" >/dev/null +} + search_resource_id() { local resource="$1" local name="$2" @@ -304,8 +399,9 @@ register_model() { local payload response model_id task_id task_response task_state payload="$(jq -nc \ --arg name "$name" \ + --arg provider "$PROVIDER_LABEL" \ --arg connector_id "$connector_id" \ - '{name:$name, description:("Ollama remote model for " + $name), function_name:"remote", connector_id:$connector_id}')" + '{name:$name, description:($provider + " remote model for " + $name), function_name:"remote", connector_id:$connector_id}')" response="$(os_request POST '/_plugins/_ml/models/_register' "$payload")" model_id="$(jq -r '.model_id // empty' <<<"$response")" task_id="$(jq -r '.task_id // empty' <<<"$response")" @@ -392,95 +488,93 @@ ensure_utility_agent() { } wait_for_opensearch -wait_for_ollama +if [[ "$PROVIDER" == "sglang" ]]; then + check_sglang + trust_sglang_endpoint +else + wait_for_ollama +fi # The single quotes deliberately preserve ML Commons ${parameters.*} # placeholders until connector inference time. # shellcheck disable=SC2016 printf -v CHAT_REQUEST_BODY \ - '{ "model": "${parameters.model}", "messages": [{"role":"system","content":"${parameters.system_prompt}"},{"role":"user","content":"Join orders with customers on customer_id and return order_id and customer name, limited to 10 rows."},{"role":"assistant","content":"","tool_calls":[{"id":"join_example","index":0,"type":"function","function":{"name":"TransferQuestionToPPLAndExecuteTool","arguments":"{\\\"index\\\":\\\"orders\\\",\\\"question\\\":\\\"Join orders with customers on customer_id. Return order_id and customer name. Limit to 10 rows.\\\"}"}}]},{"role":"tool","tool_call_id":"join_example","name":"TransferQuestionToPPLAndExecuteTool","content":"PPL: source = orders | inner join left=l right=r on l.customer_id = r.customer_id customers | fields l.order_id, r.name | head 10. Result: order_id 1, name Alice. Total rows: 1."},{"role":"assistant","content":"The join returned one row: order 1 belongs to Alice."},${parameters._chat_history:-}{"role":"user","content":"${parameters.prompt}"}${parameters._interactions:-}], "stream": false, "temperature": 0, "max_tokens": %d, "reasoning_effort": "%s"${parameters.tool_configs:-} }' \ - "$MAX_TOKENS" "$REASONING_EFFORT" + '{ "model": "${parameters.model}", "messages": [{"role":"system","content":"${parameters.system_prompt}"},{"role":"user","content":"Join orders with customers on customer_id and return order_id and customer name, limited to 10 rows."},{"role":"assistant","content":"","tool_calls":[{"id":"join_example","index":0,"type":"function","function":{"name":"TransferQuestionToPPLAndExecuteTool","arguments":"{\\\"index\\\":\\\"orders\\\",\\\"question\\\":\\\"Join orders with customers on customer_id. Return order_id and customer name. Limit to 10 rows.\\\"}"}}]},{"role":"tool","tool_call_id":"join_example","name":"TransferQuestionToPPLAndExecuteTool","content":"PPL: source = orders | inner join left=l right=r on l.customer_id = r.customer_id customers | fields l.order_id, r.name | head 10. Result: order_id 1, name Alice. Total rows: 1."},{"role":"assistant","content":"The join returned one row: order 1 belongs to Alice."},${parameters._chat_history:-}{"role":"user","content":"${parameters.prompt}"}${parameters._interactions:-}], "stream": false, "temperature": 0, "max_tokens": %d%s${parameters.tool_configs:-} }' \ + "$MAX_TOKENS" "$REASONING_OPTION" # shellcheck disable=SC2016 printf -v PPL_REQUEST_BODY \ - '{ "model": "${parameters.model}", "messages": [{"role":"system","content":"Translate the request into exactly one valid, read-only OpenSearch PPL query and return only the query, with no Markdown or explanation. Never use SQL syntax such as SELECT, FROM, JOIN ... ON, LIMIT, or column AS aliases. For every two-index join, aliases MUST be exactly l for the left index and r for the right index. The condition MUST compare l.join_field = r.join_field; never compare the same alias on both sides. Prefix every projected left field with l and every projected right field with r. Preserve every requested output field in the fields command, including a join field when the user requests it as output. Put the right index immediately after the join condition. Use fields, never select, to project columns. Use head, never limit, to cap results. Filter each side before joining when filters are requested. Finish with head 100 unless the request asks for a smaller limit. Never target system indices whose names start with a dot."},{"role":"user","content":"Join orders with customers on customer_id and return customer_id, order_id, and customer name, limited to 10 rows."},{"role":"assistant","content":"source = orders | inner join left=l right=r on l.customer_id = r.customer_id customers | fields l.customer_id, l.order_id, r.name | head 10"},{"role":"user","content":"${parameters.prompt}"}], "stream": false, "temperature": 0, "max_tokens": %d, "reasoning_effort": "%s" }' \ - "$MAX_TOKENS" "$REASONING_EFFORT" + '{ "model": "${parameters.model}", "messages": [{"role":"system","content":"Translate the request into exactly one valid, read-only OpenSearch PPL query and return only the query, with no Markdown or explanation. Never use SQL syntax such as SELECT, FROM, JOIN ... ON, LIMIT, or column AS aliases. For every two-index join, aliases MUST be exactly l for the left index and r for the right index. The condition MUST compare l.join_field = r.join_field; never compare the same alias on both sides. Prefix every projected left field with l and every projected right field with r. Preserve every requested output field in the fields command, including a join field when the user requests it as output. Put the right index immediately after the join condition. Use fields, never select, to project columns. Use head, never limit, to cap results. Filter each side before joining when filters are requested. Finish with head 100 unless the request asks for a smaller limit. Never target system indices whose names start with a dot."},{"role":"user","content":"Join orders with customers on customer_id and return customer_id, order_id, and customer name, limited to 10 rows."},{"role":"assistant","content":"source = orders | inner join left=l right=r on l.customer_id = r.customer_id customers | fields l.customer_id, l.order_id, r.name | head 10"},{"role":"user","content":"${parameters.prompt}"}], "stream": false, "temperature": 0, "max_tokens": %d%s }' \ + "$MAX_TOKENS" "$REASONING_OPTION" # shellcheck disable=SC2016 printf -v TIME_RANGE_REQUEST_BODY \ - '{ "model": "${parameters.model}", "messages": [{"role":"system","content":"Parse time constraints for an OpenSearch date picker. Follow the requested XML output format exactly and return no explanation."},{"role":"user","content":"${parameters.prompt}"}], "stream": false, "max_tokens": %d, "reasoning_effort": "%s" }' \ - "$MAX_TOKENS" "$REASONING_EFFORT" + '{ "model": "${parameters.model}", "messages": [{"role":"system","content":"Parse time constraints for an OpenSearch date picker. Follow the requested XML output format exactly and return no explanation."},{"role":"user","content":"${parameters.prompt}"}], "stream": false, "max_tokens": %d%s }' \ + "$MAX_TOKENS" "$REASONING_OPTION" # General-purpose strict-output connector used by the feature-specific agents # below. Their MLModelTool prompts define the exact response contract expected # by each OpenSearch Dashboards route. # shellcheck disable=SC2016 printf -v UTILITY_REQUEST_BODY \ - '{ "model": "${parameters.model}", "messages": [{"role":"system","content":"Follow the requested output format exactly. Do not add Markdown fences or commentary outside that format."},{"role":"user","content":"${parameters.prompt}"}], "stream": false, "max_tokens": %d, "reasoning_effort": "%s" }' \ - "$UTILITY_MAX_TOKENS" "$REASONING_EFFORT" + '{ "model": "${parameters.model}", "messages": [{"role":"system","content":"Follow the requested output format exactly. Do not add Markdown fences or commentary outside that format."},{"role":"user","content":"${parameters.prompt}"}], "stream": false, "max_tokens": %d%s }' \ + "$UTILITY_MAX_TOKENS" "$REASONING_OPTION" # The anomaly detector UI calls String.split on every multi-value property. -# Enforce strings at the Ollama decoder so the UI never receives JSON arrays. +# Enforce strings at the model decoder so the UI never receives JSON arrays. # shellcheck disable=SC2016 printf -v AD_REQUEST_BODY \ - '{ "model": "${parameters.model}", "messages": [{"role":"system","content":"Suggest anomaly detector parameters from the supplied index mapping. Follow the JSON schema exactly."},{"role":"user","content":"${parameters.prompt}"}], "stream": false, "max_tokens": %d, "reasoning_effort": "%s", "response_format":{"type":"json_schema","json_schema":{"name":"anomaly_detector_parameters","strict":true,"schema":{"type":"object","properties":{"categoryField":{"type":"string"},"aggregationField":{"type":"string"},"aggregationMethod":{"type":"string"},"dateFields":{"type":"string"}},"required":["categoryField","aggregationField","aggregationMethod","dateFields"],"additionalProperties":false}}} }' \ - "$UTILITY_MAX_TOKENS" "$REASONING_EFFORT" + '{ "model": "${parameters.model}", "messages": [{"role":"system","content":"Suggest anomaly detector parameters from the supplied index mapping. Follow the JSON schema exactly."},{"role":"user","content":"${parameters.prompt}"}], "stream": false, "max_tokens": %d%s, "response_format":{"type":"json_schema","json_schema":{"name":"anomaly_detector_parameters","strict":true,"schema":{"type":"object","properties":{"categoryField":{"type":"string"},"aggregationField":{"type":"string"},"aggregationMethod":{"type":"string"},"dateFields":{"type":"string"}},"required":["categoryField","aggregationField","aggregationMethod","dateFields"],"additionalProperties":false}}} }' \ + "$UTILITY_MAX_TOKENS" "$REASONING_OPTION" -# Qwen 3.5 is multimodal. Send notebook visualization screenshots through the +# A multimodal model is needed for notebook visualization screenshots. Use the # OpenAI-compatible image_url message format and make the model emit the nested # JSON shape expected by the Dashboards investigation plugin. # shellcheck disable=SC2016 printf -v VISUAL_REQUEST_BODY \ - '{ "model": "${parameters.model}", "messages": [{"role":"system","content":"Summarize the supplied OpenSearch visualization. Return only JSON shaped as {\\"output\\":{\\"message\\":{\\"content\\":[{\\"text\\":\\"concise summary\\"}]}}}."},{"role":"user","content":[{"type":"text","text":"Explain the notable trends, outliers, and operational meaning. Local timezone offset in minutes: ${parameters.local_time_offset}"},{"type":"image_url","image_url":{"url":"data:image/jpeg;base64,${parameters.image_base64}"}}]}], "stream": false, "max_tokens": %d, "reasoning_effort": "%s" }' \ - "$UTILITY_MAX_TOKENS" "$REASONING_EFFORT" + '{ "model": "${parameters.model}", "messages": [{"role":"system","content":"Summarize the supplied OpenSearch visualization. Return only JSON shaped as {\\"output\\":{\\"message\\":{\\"content\\":[{\\"text\\":\\"concise summary\\"}]}}}."},{"role":"user","content":[{"type":"text","text":"Explain the notable trends, outliers, and operational meaning. Local timezone offset in minutes: ${parameters.local_time_offset}"},{"type":"image_url","image_url":{"url":"data:image/jpeg;base64,${parameters.image_base64}"}}]}], "stream": false, "max_tokens": %d%s }' \ + "$UTILITY_MAX_TOKENS" "$REASONING_OPTION" CLIENT_CONFIG="$(jq -nc \ --argjson connection_timeout "$CONNECTION_TIMEOUT" \ --argjson read_timeout "$READ_TIMEOUT" \ '{max_connection:10, connection_timeout:$connection_timeout, read_timeout:$read_timeout, max_retry_times:0}')" -CHAT_CONNECTOR_PAYLOAD="$(jq -nc \ - --arg name "$CHAT_CONNECTOR_NAME" \ - --arg model "$OLLAMA_MODEL_NAME" \ - --arg endpoint "$CHAT_ENDPOINT" \ - --arg request_body "$CHAT_REQUEST_BODY" \ - --argjson client_config "$CLIENT_CONFIG" \ - '{name:$name, description:"OpenAI-compatible Ollama connector for OpenSearch conversational agents", version:"1", protocol:"http", parameters:{endpoint:$endpoint, model:$model}, credential:{ollama_key:"local"}, client_config:$client_config, actions:[{action_type:"predict", method:"POST", url:"http://${parameters.endpoint}/v1/chat/completions", headers:{"Content-Type":"application/json"}, request_body:$request_body}]}')" - -PPL_CONNECTOR_PAYLOAD="$(jq -nc \ - --arg name "$PPL_CONNECTOR_NAME" \ - --arg model "$OLLAMA_MODEL_NAME" \ - --arg request_body "$PPL_REQUEST_BODY" \ - --argjson client_config "$CLIENT_CONFIG" \ - '{name:$name, description:"Prompt-compatible Ollama connector for OpenSearch PPL query assist", version:"1", protocol:"http", parameters:{endpoint:"ollama:11434", model:$model, response_filter:"$.choices[0].message.content"}, credential:{ollama_key:"local"}, client_config:$client_config, actions:[{action_type:"predict", method:"POST", url:"http://${parameters.endpoint}/v1/chat/completions", headers:{"Content-Type":"application/json"}, request_body:$request_body}]}')" - -TIME_RANGE_CONNECTOR_PAYLOAD="$(jq -nc \ - --arg name "$TIME_RANGE_CONNECTOR_NAME" \ - --arg model "$OLLAMA_MODEL_NAME" \ - --arg request_body "$TIME_RANGE_REQUEST_BODY" \ - --argjson client_config "$CLIENT_CONFIG" \ - '{name:$name, description:"Ollama connector for OpenSearch query time-range parsing", version:"1", protocol:"http", parameters:{endpoint:"ollama:11434", model:$model, response_filter:"$.choices[0].message.content"}, credential:{ollama_key:"local"}, client_config:$client_config, actions:[{action_type:"predict", method:"POST", url:"http://${parameters.endpoint}/v1/chat/completions", headers:{"Content-Type":"application/json"}, request_body:$request_body}]}')" - -UTILITY_CONNECTOR_PAYLOAD="$(jq -nc \ - --arg name "$UTILITY_CONNECTOR_NAME" \ - --arg model "$OLLAMA_MODEL_NAME" \ - --arg request_body "$UTILITY_REQUEST_BODY" \ - --argjson client_config "$CLIENT_CONFIG" \ - '{name:$name, description:"Ollama connector for OpenSearch Dashboards feature agents", version:"1", protocol:"http", parameters:{endpoint:"ollama:11434", model:$model, response_filter:"$.choices[0].message.content"}, credential:{ollama_key:"local"}, client_config:$client_config, actions:[{action_type:"predict", method:"POST", url:"http://${parameters.endpoint}/v1/chat/completions", headers:{"Content-Type":"application/json"}, request_body:$request_body}]}')" - -AD_CONNECTOR_PAYLOAD="$(jq -nc \ - --arg name "$AD_CONNECTOR_NAME" \ - --arg model "$OLLAMA_MODEL_NAME" \ - --arg request_body "$AD_REQUEST_BODY" \ - --argjson client_config "$CLIENT_CONFIG" \ - '{name:$name, description:"Structured-output Ollama connector for anomaly detector suggestions", version:"1", protocol:"http", parameters:{endpoint:"ollama:11434", model:$model, response_filter:"$.choices[0].message.content"}, credential:{ollama_key:"local"}, client_config:$client_config, actions:[{action_type:"predict", method:"POST", url:"http://${parameters.endpoint}/v1/chat/completions", headers:{"Content-Type":"application/json"}, request_body:$request_body}]}')" - -VISUAL_CONNECTOR_PAYLOAD="$(jq -nc \ - --arg name "$VISUAL_CONNECTOR_NAME" \ - --arg model "$OLLAMA_MODEL_NAME" \ - --arg request_body "$VISUAL_REQUEST_BODY" \ - --argjson client_config "$CLIENT_CONFIG" \ - '{name:$name, description:"OpenAI-compatible Ollama connector for visualization screenshots", version:"1", protocol:"http", parameters:{endpoint:"ollama:11434", model:$model, response_filter:"$.choices[0].message.content"}, credential:{ollama_key:"local"}, client_config:$client_config, actions:[{action_type:"predict", method:"POST", url:"http://${parameters.endpoint}/v1/chat/completions", headers:{"Content-Type":"application/json"}, request_body:$request_body}]}')" +# All feature connectors share provider, authentication, and timeout settings. +# Ollama chat retains its compatibility proxy; SGLang uses its native API. +connector_payload() { + local name="$1" + local purpose="$2" + local base_url="$3" + local request_body="$4" + local response_filter="${5-}" + jq -nc \ + --arg name "$name" \ + --arg description "$PROVIDER_LABEL connector for $purpose" \ + --arg model "$MODEL_NAME" \ + --arg base_url "$base_url" \ + --arg api_key "$API_KEY" \ + --arg response_filter "$response_filter" \ + --arg request_body "$request_body" \ + --argjson client_config "$CLIENT_CONFIG" \ + '{name:$name, description:$description, version:"1", protocol:"http", + parameters:({base_url:$base_url, model:$model} + + if $response_filter == "" then {} else {response_filter:$response_filter} end), + credential:{api_key:(if $api_key == "" then "local" else $api_key end)}, + client_config:$client_config, + actions:[{action_type:"predict", method:"POST", + url:"${parameters.base_url}/chat/completions", + headers:({"Content-Type":"application/json"} + + if $api_key == "" then {} else {Authorization:"Bearer ${credential.api_key}"} end), + request_body:$request_body}]}' +} + +CHAT_CONNECTOR_PAYLOAD="$(connector_payload "$CHAT_CONNECTOR_NAME" "OpenSearch conversational agents" "$CHAT_BASE_URL" "$CHAT_REQUEST_BODY")" +PPL_CONNECTOR_PAYLOAD="$(connector_payload "$PPL_CONNECTOR_NAME" "OpenSearch PPL query assist" "$BASE_URL" "$PPL_REQUEST_BODY" '$.choices[0].message.content')" +TIME_RANGE_CONNECTOR_PAYLOAD="$(connector_payload "$TIME_RANGE_CONNECTOR_NAME" "OpenSearch query time-range parsing" "$BASE_URL" "$TIME_RANGE_REQUEST_BODY" '$.choices[0].message.content')" +UTILITY_CONNECTOR_PAYLOAD="$(connector_payload "$UTILITY_CONNECTOR_NAME" "OpenSearch Dashboards feature agents" "$BASE_URL" "$UTILITY_REQUEST_BODY" '$.choices[0].message.content')" +AD_CONNECTOR_PAYLOAD="$(connector_payload "$AD_CONNECTOR_NAME" "anomaly detector suggestions" "$BASE_URL" "$AD_REQUEST_BODY" '$.choices[0].message.content')" +VISUAL_CONNECTOR_PAYLOAD="$(connector_payload "$VISUAL_CONNECTOR_NAME" "visualization screenshots" "$BASE_URL" "$VISUAL_REQUEST_BODY" '$.choices[0].message.content')" CHAT_CONNECTOR_ID="$(upsert_connector "$CHAT_CONNECTOR_NAME" "$CHAT_CONNECTOR_PAYLOAD")" CHAT_MODEL_ID="$(ensure_model "$CHAT_MODEL_NAME" "$CHAT_CONNECTOR_ID")" @@ -616,15 +710,15 @@ Data schema: ${parameters.dataSchema} Sample data: ${parameters.sampleData} Use field names that exist in the supplied data. Return only one valid JSON object. Do not include width, height, or inline data.' -DATA2SUMMARY_AGENT_ID="$(ensure_utility_agent "Qwen Data Summary Agent" "Summarize Discover result samples" "os_data2summary" "$DATA2SUMMARY_PROMPT")" -LOG_DATA2SUMMARY_AGENT_ID="$(ensure_utility_agent "Qwen Log Data Summary Agent" "Summarize Discover log result samples" "os_data2summary" "$LOG_DATA2SUMMARY_PROMPT")" -INDEX_TYPE_AGENT_ID="$(ensure_utility_agent "Qwen Index Type Detection Agent" "Classify whether an index contains operational log data" "os_index_type_detect" "$INDEX_TYPE_PROMPT")" -QUERY_RESPONSE_SUMMARY_AGENT_ID="$(ensure_utility_agent "Qwen Query Response Summary Agent" "Summarize successful PPL query results" "query_assist" "$QUERY_RESPONSE_SUMMARY_PROMPT")" -QUERY_ERROR_SUMMARY_AGENT_ID="$(ensure_utility_agent "Qwen Query Error Summary Agent" "Explain failed PPL queries" "query_assist" "$QUERY_ERROR_SUMMARY_PROMPT")" -ALERT_SUMMARY_AGENT_ID="$(ensure_utility_agent "Qwen Alert Summary Agent" "Summarize alert context" "os_summary" "$ALERT_SUMMARY_PROMPT")" -LOG_ALERT_SUMMARY_AGENT_ID="$(ensure_utility_agent "Qwen Log Alert Summary Agent" "Summarize log alert context" "os_summary" "$LOG_ALERT_SUMMARY_PROMPT")" -TEXT2VEGA_AGENT_ID="$(ensure_utility_agent "Qwen Text to Vega Agent" "Create Vega-Lite specifications from query results" "os_text2vega" "$TEXT2VEGA_PROMPT")" -TEXT2VEGA_INSTRUCTIONS_AGENT_ID="$(ensure_utility_agent "Qwen Text to Vega Instructions Agent" "Modify Vega-Lite specifications using user instructions" "os_text2vega" "$TEXT2VEGA_INSTRUCTIONS_PROMPT")" +DATA2SUMMARY_AGENT_ID="$(ensure_utility_agent "$AGENT_NAME_PREFIX Data Summary Agent" "Summarize Discover result samples" "os_data2summary" "$DATA2SUMMARY_PROMPT")" +LOG_DATA2SUMMARY_AGENT_ID="$(ensure_utility_agent "$AGENT_NAME_PREFIX Log Data Summary Agent" "Summarize Discover log result samples" "os_data2summary" "$LOG_DATA2SUMMARY_PROMPT")" +INDEX_TYPE_AGENT_ID="$(ensure_utility_agent "$AGENT_NAME_PREFIX Index Type Detection Agent" "Classify whether an index contains operational log data" "os_index_type_detect" "$INDEX_TYPE_PROMPT")" +QUERY_RESPONSE_SUMMARY_AGENT_ID="$(ensure_utility_agent "$AGENT_NAME_PREFIX Query Response Summary Agent" "Summarize successful PPL query results" "query_assist" "$QUERY_RESPONSE_SUMMARY_PROMPT")" +QUERY_ERROR_SUMMARY_AGENT_ID="$(ensure_utility_agent "$AGENT_NAME_PREFIX Query Error Summary Agent" "Explain failed PPL queries" "query_assist" "$QUERY_ERROR_SUMMARY_PROMPT")" +ALERT_SUMMARY_AGENT_ID="$(ensure_utility_agent "$AGENT_NAME_PREFIX Alert Summary Agent" "Summarize alert context" "os_summary" "$ALERT_SUMMARY_PROMPT")" +LOG_ALERT_SUMMARY_AGENT_ID="$(ensure_utility_agent "$AGENT_NAME_PREFIX Log Alert Summary Agent" "Summarize log alert context" "os_summary" "$LOG_ALERT_SUMMARY_PROMPT")" +TEXT2VEGA_AGENT_ID="$(ensure_utility_agent "$AGENT_NAME_PREFIX Text to Vega Agent" "Create Vega-Lite specifications from query results" "os_text2vega" "$TEXT2VEGA_PROMPT")" +TEXT2VEGA_INSTRUCTIONS_AGENT_ID="$(ensure_utility_agent "$AGENT_NAME_PREFIX Text to Vega Instructions Agent" "Modify Vega-Lite specifications using user instructions" "os_text2vega" "$TEXT2VEGA_INSTRUCTIONS_PROMPT")" AD_CONNECTOR_ID="$(upsert_connector "$AD_CONNECTOR_NAME" "$AD_CONNECTOR_PAYLOAD")" AD_MODEL_ID="$(ensure_model "$AD_MODEL_NAME" "$AD_CONNECTOR_ID")" @@ -640,17 +734,19 @@ Choose one to three suitable numeric or countable aggregation fields, matching a Return only valid JSON in this exact shape: {"categoryField":"service.keyword","aggregationField":"latency,error_count","aggregationMethod":"avg,sum","dateFields":"@timestamp"}. Every property value must be a JSON string, never an array. Represent multiple fields or methods inside one comma-delimited string. Use an empty categoryField when none is suitable. The aggregationField and aggregationMethod comma-delimited lists must have equal lengths.' SUGGEST_AD_AGENT_PAYLOAD="$(jq -nc \ + --arg name "$AGENT_NAME_PREFIX Anomaly Detector Suggestion Agent" \ --arg model_id "$AD_MODEL_ID" \ --arg prompt "$SUGGEST_AD_PROMPT" \ - '{name:"Qwen Anomaly Detector Suggestion Agent", description:"Suggest anomaly detector fields from an index mapping", type:"flow", app_type:"os_suggest_ad", tools:[{type:"IndexMappingTool", name:"IndexMappingTool", include_output_in_agent_response:false, parameters:{index:"${parameters.index}"}},{type:"MLModelTool", name:"SuggestAnomalyDetectorTool", include_output_in_agent_response:true, parameters:{model_id:$model_id, model_type:"OPENAI", prompt:$prompt, response_filter:"$.choices[0].message.content"}}]}')" -SUGGEST_AD_AGENT_ID="$(upsert_agent "Qwen Anomaly Detector Suggestion Agent" "$SUGGEST_AD_AGENT_PAYLOAD")" + '{name:$name, description:"Suggest anomaly detector fields from an index mapping", type:"flow", app_type:"os_suggest_ad", tools:[{type:"IndexMappingTool", name:"IndexMappingTool", include_output_in_agent_response:false, parameters:{index:"${parameters.index}"}},{type:"MLModelTool", name:"SuggestAnomalyDetectorTool", include_output_in_agent_response:true, parameters:{model_id:$model_id, model_type:"OPENAI", prompt:$prompt, response_filter:"$.choices[0].message.content"}}]}')" +SUGGEST_AD_AGENT_ID="$(upsert_agent "$AGENT_NAME_PREFIX Anomaly Detector Suggestion Agent" "$SUGGEST_AD_AGENT_PAYLOAD")" VISUAL_CONNECTOR_ID="$(upsert_connector "$VISUAL_CONNECTOR_NAME" "$VISUAL_CONNECTOR_PAYLOAD")" VISUAL_MODEL_ID="$(ensure_model "$VISUAL_MODEL_NAME" "$VISUAL_CONNECTOR_ID")" VISUAL_SUMMARY_AGENT_PAYLOAD="$(jq -nc \ + --arg name "$AGENT_NAME_PREFIX Visualization Summary Agent" \ --arg model_id "$VISUAL_MODEL_ID" \ - '{name:"Qwen Visualization Summary Agent", description:"Summarize OpenSearch notebook visualization screenshots", type:"flow", app_type:"os_visualization_summary", tools:[{type:"MLModelTool", name:"VisualizationSummaryTool", include_output_in_agent_response:true, parameters:{model_id:$model_id, model_type:"OPENAI", response_filter:"$.choices[0].message.content"}}]}')" -VISUAL_SUMMARY_AGENT_ID="$(upsert_agent "Qwen Visualization Summary Agent" "$VISUAL_SUMMARY_AGENT_PAYLOAD")" + '{name:$name, description:"Summarize OpenSearch notebook visualization screenshots", type:"flow", app_type:"os_visualization_summary", tools:[{type:"MLModelTool", name:"VisualizationSummaryTool", include_output_in_agent_response:true, parameters:{model_id:$model_id, model_type:"OPENAI", response_filter:"$.choices[0].message.content"}}]}')" +VISUAL_SUMMARY_AGENT_ID="$(upsert_agent "$AGENT_NAME_PREFIX Visualization Summary Agent" "$VISUAL_SUMMARY_AGENT_PAYLOAD")" write_ml_config os_chat os_chat_root_agent "$ROOT_AGENT_ID" write_ml_config os_query_assist_ppl os_query_assist_ppl_agent "$PPL_AGENT_ID" @@ -667,7 +763,7 @@ write_ml_config os_text2vega_with_instructions os_text2vega_agent "$TEXT2VEGA_IN write_ml_config os_suggest_ad os_suggest_ad_agent "$SUGGEST_AD_AGENT_ID" write_ml_config os_visualization_summary os_visualization_summary_agent "$VISUAL_SUMMARY_AGENT_ID" -log "Bootstrap complete" +log "Bootstrap complete (provider=$PROVIDER, model=$MODEL_NAME)" printf ' chat_connector_id=%s\n' "$CHAT_CONNECTOR_ID" printf ' chat_model_id=%s\n' "$CHAT_MODEL_ID" printf ' chat_agent_id=%s\n' "$CHAT_AGENT_ID" diff --git a/scripts/tests/test_opensearch_ai_init.py b/scripts/tests/test_opensearch_ai_init.py new file mode 100644 index 000000000..2c3590885 --- /dev/null +++ b/scripts/tests/test_opensearch_ai_init.py @@ -0,0 +1,235 @@ +"""Exercise the bootstrap against a fake Docker/OpenSearch API; no live writes.""" + +import json +import os +from pathlib import Path +import re +import shutil +import subprocess +import tempfile +import unittest + + +PROJECT_DIR = Path(__file__).resolve().parents[2] +TRUST_KEY = "plugins.ml_commons.trusted_connector_endpoints_regex" +PRIVATE_KEY = "plugins.ml_commons.trusted_connector_private_endpoints_regex" +PRIVATE_ENABLED = "plugins.ml_commons.connector.private_ip_enabled" + +FAKE_DOCKER = r'''#!/usr/bin/env python3 +import json +import os +from pathlib import Path +import sys +from urllib.parse import urlsplit + +state_path = Path(os.environ["FAKE_DOCKER_STATE"]) +state = json.loads(state_path.read_text()) +args = sys.argv[1:] +state["calls"].append(args) + +def reply(body=None, status=200, exit_code=0): + state_path.write_text(json.dumps(state)) + if body is not None: + print(json.dumps(body)) + if "--write-out" in args: + print(status) + sys.exit(exit_code) + +if args[0] == "inspect": + state_path.write_text(json.dumps(state)) + print("true") + sys.exit(0) +if "ollama" in args: + reply() +if "curl" not in args: + raise RuntimeError(args) +url = args[-1] +method = args[args.index("--request") + 1] if "--request" in args else "GET" +body = json.load(sys.stdin) if "--data-binary" in args else None +state["requests"].append({"method": method, "url": url, "body": body}) +path = urlsplit(url).path +if path.endswith("/v1/models"): + reply(json.loads(os.environ.get("FAKE_MODELS", '{"data":[{"id":"test-model"}]}')), + exit_code=int(os.environ.get("FAKE_MODEL_EXIT", "0"))) +if path == "/_cluster/health": + reply({"status": "yellow"}) +if path == "/_cluster/settings": + if method == "GET": + reply(state["settings"]) + for scope, values in body.items(): + state["settings"].setdefault(scope, {}).update(values) + reply({"acknowledged": True}) +if path.startswith("/.plugins-ml-config/_doc/"): + state["configs"][path.rsplit("/", 1)[-1]] = body + reply({"result": "updated"}) +parts = path.strip("/").split("/") +if parts[:2] != ["_plugins", "_ml"]: + raise RuntimeError(path) +resource, action = parts[2:4] +resources = state["resources"][resource] +if action == "_search": + reply({"hits": {"hits": [{"_id": key, "_source": value} + for key, value in resources.items()]}}) +if action in ("_create", "_register"): + resource_id = f"{resource}-{state['next_id']}" + state["next_id"] += 1 + resources[resource_id] = body + reply({{"connectors": "connector_id", "models": "model_id", "agents": "agent_id"}[resource]: resource_id}) +if len(parts) > 4: + resources[action]["model_state"] = "DEPLOYED" if parts[4] == "_deploy" else "REGISTERED" + reply({"status": "COMPLETED"}) +if method == "GET": + reply(resources[action]) +resources[action].update(body) +reply({"result": "updated"}) +''' + + +@unittest.skipUnless(shutil.which("jq"), "jq is required") +class OpenSearchAIInitTest(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + (self.root / "scripts").mkdir() + (self.root / "deploy").mkdir() + self.script = self.root / "scripts/opensearch-ai-init.sh" + shutil.copy(PROJECT_DIR / "scripts/opensearch-ai-init.sh", self.script) + (self.root / "deploy/ollama.env").write_text( + "OLLAMA_MODEL=file-model\nOPENSEARCH_AI_CHAT_ENDPOINT=ollama-openai-proxy:11435\n" + ) + shutil.copy(PROJECT_DIR / "deploy/sglang.env", self.root / "deploy/sglang.env") + docker = self.root / "docker" + docker.write_text(FAKE_DOCKER) + docker.chmod(0o755) + self.state_path = self.root / "state.json" + self.state_path.write_text(json.dumps({ + "calls": [], "requests": [], "configs": {}, "next_id": 1, + "resources": {"connectors": {}, "models": {}, "agents": {}}, + "settings": { + "persistent": {TRUST_KEY: ["^https://existing.example/.*$"]}, + "transient": {TRUST_KEY: ["^https://transient.example/.*$"], PRIVATE_ENABLED: "false"}, + "defaults": {PRIVATE_KEY: ["^http://ollama:11434/.*$"]}, + }, + })) + self.env = { + key: value for key, value in os.environ.items() + if not key.startswith(("OPENSEARCH_AI_", "SGLANG_", "OLLAMA_")) + } + self.env.update({ + "PATH": f"{self.root}:{os.environ['PATH']}", + "FAKE_DOCKER_STATE": str(self.state_path), + "OPENSEARCH_AI_OPENSEARCH_PASSWORD": "test-password", + "OPENSEARCH_AI_PROVIDER": "sglang", + }) + + def run_init(self, **overrides): + return subprocess.run( + ["bash", str(self.script)], env={**self.env, **overrides}, + text=True, capture_output=True, timeout=45, + ) + + def state(self): + return json.loads(self.state_path.read_text()) + + def assert_success(self, result): + self.assertEqual(result.returncode, 0, result.stderr) + + def request_body(self, connector): + body = connector["actions"][0]["request_body"] + # Substitute ML Commons placeholders to validate the emitted JSON too. + body = re.sub( + r"\$\{parameters\.([^}:]+)(?::-)?\}", + lambda match: "" if match[1] in ("_chat_history", "_interactions", "tool_configs") else "example", + body, + ) + return json.loads(body) + + def test_sglang_routes_every_feature_and_discovers_model(self): + self.assert_success(self.run_init()) + state = self.state() + connectors = list(state["resources"]["connectors"].values()) + self.assertEqual(len(connectors), 6) + self.assertEqual(len(state["configs"]), 14) + for connector in connectors: + self.assertTrue(connector["name"].startswith("SGLang")) + self.assertEqual(connector["parameters"]["model"], "test-model") + self.assertEqual(connector["parameters"]["base_url"], "http://host.docker.internal:30000/v1") + self.assertNotIn("Authorization", connector["actions"][0]["headers"]) + self.assertNotIn("reasoning_effort", self.request_body(connector)) + self.assertTrue(all(a["name"].startswith("SGLang") for a in state["resources"]["agents"].values())) + self.assertFalse(any("cogstack-ollama" in " ".join(call) for call in state["calls"])) + + def test_exact_trust_preserves_persistent_defaults_and_transient(self): + self.assert_success(self.run_init(SGLANG_BASE_URL="https://inference.example:8443/team/v1/")) + settings = self.state()["settings"] + pattern = r"^\Qhttps://inference.example:8443/team/v1/chat/completions\E$" + self.assertEqual(settings["persistent"][TRUST_KEY], sorted(["^https://existing.example/.*$", pattern])) + self.assertEqual(settings["persistent"][PRIVATE_KEY], sorted(["^http://ollama:11434/.*$", pattern])) + self.assertEqual(settings["transient"][TRUST_KEY], sorted(["^https://transient.example/.*$", pattern])) + self.assertIs(settings["persistent"][PRIVATE_ENABLED], True) + self.assertIs(settings["transient"][PRIVATE_ENABLED], True) + + def test_authenticated_model_and_caller_overrides(self): + (self.root / "deploy/sglang.env").write_text( + "OPENSEARCH_AI_PROVIDER=ollama\nSGLANG_BASE_URL=http://file-host:1/v1\n" + "SGLANG_MODEL=file-model\nSGLANG_API_KEY=file-key\n" + ) + self.assert_success(self.run_init( + SGLANG_BASE_URL="http://lan-host:8000/v1", SGLANG_MODEL="test-model", + SGLANG_API_KEY="test-token", OPENSEARCH_AI_REASONING_EFFORT="low", + OPENSEARCH_AI_MAX_TOKENS="2048", + )) + state = self.state() + for connector in state["resources"]["connectors"].values(): + self.assertEqual(connector["parameters"]["base_url"], "http://lan-host:8000/v1") + self.assertEqual(connector["credential"], {"api_key": "test-token"}) + self.assertEqual(connector["actions"][0]["headers"]["Authorization"], "Bearer ${credential.api_key}") + self.assertEqual(self.request_body(connector)["reasoning_effort"], "low") + probe = next(call for call in state["calls"] if call[-1].endswith("/v1/models")) + self.assertIn("Authorization: Bearer test-token", probe) + chat = next(c for c in state["resources"]["connectors"].values() if c["name"] == "SGLang Agent Connector") + self.assertEqual(self.request_body(chat)["max_tokens"], 2048) + + def test_rerun_updates_resources_and_switches_endpoint(self): + self.assert_success(self.run_init()) + before = self.state() + self.assert_success(self.run_init(SGLANG_BASE_URL="http://new-host:8000/v1")) + after = self.state() + self.assertEqual(after["next_id"], before["next_id"]) + self.assertEqual(after["configs"], before["configs"]) + for connector in after["resources"]["connectors"].values(): + self.assertEqual(connector["parameters"]["base_url"], "http://new-host:8000/v1") + self.assertTrue(all(m["model_state"] == "DEPLOYED" for m in after["resources"]["models"].values())) + + def test_ollama_defaults_and_proxy_are_preserved(self): + self.assert_success(self.run_init(OPENSEARCH_AI_PROVIDER="ollama", OLLAMA_MODEL="caller-model")) + state = self.state() + for connector in state["resources"]["connectors"].values(): + self.assertEqual(connector["parameters"]["model"], "caller-model") + self.assertEqual(self.request_body(connector)["reasoning_effort"], "none") + expected = "http://ollama-openai-proxy:11435/v1" if connector["name"] == "Ollama Qwen Agent Connector" else "http://ollama:11434/v1" + self.assertEqual(connector["parameters"]["base_url"], expected) + self.assertTrue(all(a["name"].startswith("Qwen") for a in state["resources"]["agents"].values())) + self.assertFalse(any(r["method"] == "PUT" and r["url"].endswith("/_cluster/settings") for r in state["requests"])) + + def test_invalid_configuration_fails_before_writes(self): + cases = [ + {"SGLANG_MODEL": "missing"}, + {"FAKE_MODELS": '{"data":[{"id":"a"},{"id":"b"}]}'}, + {"FAKE_MODELS": '{"data":[]}'}, + {"FAKE_MODEL_EXIT": "22"}, + {"SGLANG_BASE_URL": "http://host:30000"}, + {"SGLANG_BASE_URL": "http://user:pass@host:30000/v1"}, + {"OPENSEARCH_AI_PROVIDER": "invalid"}, + ] + for case in cases: + with self.subTest(case=case): + result = self.run_init(**case) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(self.state()["resources"]["connectors"], {}) + self.assertFalse(any(r["method"] in ("PUT", "POST") for r in self.state()["requests"])) + + +if __name__ == "__main__": + unittest.main() diff --git a/services/kibana/config/opensearch.yml b/services/kibana/config/opensearch.yml index f522023e8..61c35bb39 100644 --- a/services/kibana/config/opensearch.yml +++ b/services/kibana/config/opensearch.yml @@ -137,6 +137,8 @@ opensearch.requestHeadersAllowlist: ["securitytenant", "Authorization"] ml_commons_dashboards.enabled: true assistant.chat.enabled: true assistant.incontextInsight.enabled: true +assistant.next.enabled: true +assistant.text2viz.enabled: true observability.query_assist.enabled: true savedObjects.permission.enabled: true From 9b3fc57088aeebf4a20e20c8fe88535632dc3fe8 Mon Sep 17 00:00:00 2001 From: vladd-bit Date: Thu, 29 Oct 2026 10:23:09 +0000 Subject: [PATCH 10/28] Deploy: pinned image versions across docker compose + env files. --- .github/workflows/compose_config.yml | 25 ++++ .github/workflows/submodule_sync.yml | 66 +++------ README.md | 6 +- deploy/Makefile | 10 +- deploy/charts/nifi/templates/statefulset.yaml | 19 +-- deploy/charts/nifi/values.yaml | 1 + .../charts/opensearch/templates/_helpers.tpl | 9 ++ .../templates/deployment-dashboards.yaml | 2 +- .../templates/statefulset-opensearch.yaml | 4 +- deploy/charts/opensearch/values.yaml | 2 + deploy/charts/postgresql/values.yaml | 2 +- deploy/database.env | 2 +- deploy/elasticsearch.env | 11 +- deploy/general.env | 2 +- deploy/gitea.env | 2 +- deploy/nginx.env | 3 + deploy/nifi.env | 8 +- deploy/ollama.env | 4 +- deploy/openwebui.env | 5 +- deploy/services.dev.yml | 8 +- deploy/services.yml | 31 ++-- deploy/sglang.env | 2 +- deploy/telemetry.env | 4 +- docs/deploy/air-gapped.md | 2 +- docs/deploy/deployment.md | 6 +- docs/deploy/main.md | 4 +- docs/deploy/reproducible.md | 132 ++++++++++++++++++ docs/index.md | 6 +- docs/nifi/main.md | 14 +- mkdocs.yml | 1 + nifi/Dockerfile | 6 +- scripts/git_init_submodules_in_repo.sh | 7 + scripts/git_update_submodules_in_repo.sh | 52 +++---- scripts/tests/check_deployment_images.py | 115 +++++++++++++++ scripts/tests/test_deployment_images.py | 44 ++++++ scripts/tests/test_submodule_checkout.py | 123 ++++++++++++++++ security/scripts/create_es_native_certs.sh | 5 + services/nginx/Dockerfile | 2 +- services/ollama-openai-proxy/Dockerfile | 2 +- 39 files changed, 606 insertions(+), 143 deletions(-) create mode 100644 docs/deploy/reproducible.md create mode 100755 scripts/git_init_submodules_in_repo.sh create mode 100755 scripts/tests/check_deployment_images.py create mode 100755 scripts/tests/test_deployment_images.py create mode 100755 scripts/tests/test_submodule_checkout.py diff --git a/.github/workflows/compose_config.yml b/.github/workflows/compose_config.yml index 62827643a..2463afef5 100644 --- a/.github/workflows/compose_config.yml +++ b/.github/workflows/compose_config.yml @@ -7,6 +7,13 @@ on: - "deploy/*.env" - "deploy/export_env_vars.sh" - "deploy/services*.yml" + - "deploy/charts/**" + - "deploy/helm/**" + - "nifi/Dockerfile" + - "services/nginx/Dockerfile" + - "services/ollama-openai-proxy/Dockerfile" + - "scripts/tests/check_deployment_images.py" + - "scripts/tests/test_deployment_images.py" - "security/env/*.env" - ".github/workflows/compose_config.yml" pull_request: @@ -14,6 +21,13 @@ on: - "deploy/*.env" - "deploy/export_env_vars.sh" - "deploy/services*.yml" + - "deploy/charts/**" + - "deploy/helm/**" + - "nifi/Dockerfile" + - "services/nginx/Dockerfile" + - "services/ollama-openai-proxy/Dockerfile" + - "scripts/tests/check_deployment_images.py" + - "scripts/tests/test_deployment_images.py" - "security/env/*.env" - ".github/workflows/compose_config.yml" workflow_dispatch: @@ -32,6 +46,11 @@ jobs: steps: - uses: actions/checkout@v7 + - name: Set up Helm + uses: azure/setup-helm@v5.0.1 + with: + version: v4.3.1 + - name: Validate Compose files shell: bash run: | @@ -39,3 +58,9 @@ jobs: source deploy/export_env_vars.sh docker compose -f deploy/services.yml config --quiet docker compose -f deploy/services.dev.yml config --quiet + + - name: Verify deployment image pins + run: python3 -B scripts/tests/check_deployment_images.py --helm + + - name: Test image drift checks + run: python3 -B -m unittest discover -s scripts/tests -p test_deployment_images.py -v diff --git a/.github/workflows/submodule_sync.yml b/.github/workflows/submodule_sync.yml index 090c7c938..5c6bed1a6 100644 --- a/.github/workflows/submodule_sync.yml +++ b/.github/workflows/submodule_sync.yml @@ -1,12 +1,9 @@ -name: submodules-sync +name: submodules-pinned on: - push: branches: ['**'] - tags: ['v*.*.*'] - release: - types: [published] + pull_request: workflow_dispatch: concurrency: @@ -14,47 +11,30 @@ concurrency: cancel-in-progress: true permissions: - contents: write + contents: read jobs: - sync: - name: 'Submodules Sync' + verify: + name: Verify recorded submodule commits runs-on: ubuntu-latest - - # Use the Bash shell regardless whether the GitHub Actions runner is ubuntu-latest, macos-latest, or windows-latest - defaults: - run: - shell: bash - steps: + - uses: actions/checkout@v7 + with: + submodules: recursive - - name: Checkout - uses: actions/checkout@v7 - with: - token: ${{ github.token }} - submodules: recursive - fetch-depth: 0 - - - name: Sync submodules using repository script - id: sync_submodules - run: | - set -euo pipefail - before_sha="$(git rev-parse HEAD)" + - name: Initialize recorded commits + run: ./scripts/git_init_submodules_in_repo.sh - # script may create a commit when submodule pointers change - git config user.name "Git bot" - git config user.email "bot@noreply.github.com" - ./scripts/git_update_submodules_in_repo.sh - - after_sha="$(git rev-parse HEAD)" - if [ "$before_sha" != "$after_sha" ]; then - echo "changed=true" >> "$GITHUB_OUTPUT" - git show --no-patch --oneline "$after_sha" - else - echo "changed=false" >> "$GITHUB_OUTPUT" - echo "No submodule pointer changes." - fi - - - name: Push sync commit - if: startsWith(github.ref, 'refs/heads/') && steps.sync_submodules.outputs.changed == 'true' - run: git push origin HEAD:${GITHUB_REF##refs/heads/} + - name: Verify recursive checkout + shell: bash + run: | + set -euo pipefail + status="$(git submodule status --recursive)" + printf '%s\n' "$status" + if grep -Eq '^[-+U]' <<< "$status"; then + echo "Submodules do not match the recorded commits." >&2 + exit 1 + fi + + - name: Test initialization and explicit upgrades + run: python3 -B -m unittest discover -s scripts/tests -p test_submodule_checkout.py -v diff --git a/README.md b/README.md index dc433aabd..99f39ee8e 100644 --- a/README.md +++ b/README.md @@ -52,12 +52,16 @@ This table describes repository layout. For setup and operations, use the deploy ```bash # from repository root git lfs pull -make -C deploy git-update-submodules +make -C deploy git-init-submodules make -C deploy help make -C deploy init-security make -C deploy start-data-infra ``` +Setup uses the recorded submodule commits and pinned container images. See +[Repeatable deployments](./docs/deploy/reproducible.md) for explicit upgrades +and local image builds. + After services start: - NiFi: `https://localhost:8443` diff --git a/deploy/Makefile b/deploy/Makefile index a7a5ba5c9..c4ab2dd21 100644 --- a/deploy/Makefile +++ b/deploy/Makefile @@ -91,9 +91,14 @@ git-freeze-security: ## Apply restricted git settings for safer shell operations git-unfreeze-security: ## Revert restricted git settings @../scripts/git_unfreeze_security.sh -git-update-submodules: ## Update submodules recursively in this repository +git-init-submodules: ## Initialize submodules at the commits recorded by this checkout + @../scripts/git_init_submodules_in_repo.sh + +git-update-submodules: ## Explicitly upgrade direct submodules for review (never stages or commits) @../scripts/git_update_submodules_in_repo.sh +.PHONY: git-init-submodules git-update-submodules + load-env: ## Load variables from export_env_vars.sh in a subshell $(WITH_ENV) echo "Environment variables loaded." @@ -310,9 +315,10 @@ start-nifi-dev-build: ## Build and start NiFi dev services build-nifi-image: ## Build the local NiFi image without starting services $(WITH_ENV) docker build \ --build-arg NIFI_VERSION="$${NIFI_VERSION:-2.12.0}" \ + --build-arg NIFI_BASE_IMAGE="$${NIFI_BASE_IMAGE}" \ --build-arg NIFI_UID="$${NIFI_UID:-1000}" \ --build-arg NIFI_GID="$${NIFI_GID:-1000}" \ - --tag "$${NIFI_DOCKER_IMAGE:-cogstacksystems/cogstack-nifi:latest}" \ + --tag "$${NIFI_LOCAL_DOCKER_IMAGE:-cogstack/cogstack-nifi:local}" \ --file ../nifi/Dockerfile \ .. diff --git a/deploy/charts/nifi/templates/statefulset.yaml b/deploy/charts/nifi/templates/statefulset.yaml index d3b24f353..d8612abf9 100644 --- a/deploy/charts/nifi/templates/statefulset.yaml +++ b/deploy/charts/nifi/templates/statefulset.yaml @@ -2,17 +2,12 @@ {{- fail "nifi.cluster.enabled must be true when replicaCount is greater than 1" -}} {{- end -}} {{- $envData := include "cogstack-nifi.parsedEnvFile" . | fromYaml | default (dict) -}} -{{- $imageRepository := .Values.image.repository -}} -{{- $imageTag := .Values.image.tag -}} -{{- if hasKey $envData "NIFI_DOCKER_IMAGE" -}} -{{- $image := index $envData "NIFI_DOCKER_IMAGE" -}} -{{- $repoWithoutTag := regexReplaceAll ":[^:/]+$" $image "" -}} -{{- if ne $repoWithoutTag $image -}} -{{- $imageRepository = $repoWithoutTag -}} -{{- $imageTag = regexFind "[^:]+$" $image -}} -{{- else -}} -{{- $imageRepository = $image -}} +{{- $image := printf "%s:%s" .Values.image.repository .Values.image.tag -}} +{{- if .Values.image.digest -}} +{{- $image = printf "%s@%s" .Values.image.repository .Values.image.digest -}} {{- end -}} +{{- if hasKey $envData "NIFI_DOCKER_IMAGE" -}} +{{- $image = index $envData "NIFI_DOCKER_IMAGE" -}} {{- end -}} {{- $nifiHome := .Values.nifi.home -}} {{- $httpsPort := int .Values.nifi.web.httpsPort -}} @@ -138,7 +133,7 @@ spec: fsGroup: {{ $runAsGroup }} initContainers: - name: bootstrap-nifi-runtime - image: "{{ $imageRepository }}:{{ $imageTag }}" + image: {{ $image | quote }} imagePullPolicy: {{ .Values.image.pullPolicy }} securityContext: {{- if .Values.initSecurityContext }} @@ -362,7 +357,7 @@ spec: mountPath: /work/user_schemas containers: - name: nifi - image: "{{ $imageRepository }}:{{ $imageTag }}" + image: {{ $image | quote }} imagePullPolicy: {{ .Values.image.pullPolicy }} securityContext: {{- with omit .Values.securityContext "runAsUser" "runAsGroup" }} diff --git a/deploy/charts/nifi/values.yaml b/deploy/charts/nifi/values.yaml index a7e09e4d5..eb09d9e65 100644 --- a/deploy/charts/nifi/values.yaml +++ b/deploy/charts/nifi/values.yaml @@ -6,6 +6,7 @@ replicaCount: 1 image: repository: cogstacksystems/cogstack-nifi tag: latest + digest: sha256:f3f0ce9c23f9e503042f646675ddef372acda5cffc2491f07a38996a95a948dd pullPolicy: IfNotPresent envFile: diff --git a/deploy/charts/opensearch/templates/_helpers.tpl b/deploy/charts/opensearch/templates/_helpers.tpl index b0ff02bd9..1ab4cd605 100644 --- a/deploy/charts/opensearch/templates/_helpers.tpl +++ b/deploy/charts/opensearch/templates/_helpers.tpl @@ -1,3 +1,12 @@ +{{/* Prefer the immutable digest when supplied. */}} +{{- define "cogstack-opensearch.image" -}} +{{- if .digest -}} +{{- printf "%s@%s" .repository .digest -}} +{{- else -}} +{{- printf "%s:%s" .repository .tag -}} +{{- end -}} +{{- end -}} + {{/* Chart name */}} {{- define "cogstack-opensearch.name" -}} {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} diff --git a/deploy/charts/opensearch/templates/deployment-dashboards.yaml b/deploy/charts/opensearch/templates/deployment-dashboards.yaml index df46ebeae..806cdb40e 100644 --- a/deploy/charts/opensearch/templates/deployment-dashboards.yaml +++ b/deploy/charts/opensearch/templates/deployment-dashboards.yaml @@ -38,7 +38,7 @@ spec: spec: containers: - name: dashboards - image: "{{ .Values.dashboards.image.repository }}:{{ .Values.dashboards.image.tag }}" + image: {{ include "cogstack-opensearch.image" .Values.dashboards.image | quote }} imagePullPolicy: {{ .Values.dashboards.image.pullPolicy }} ports: - name: http diff --git a/deploy/charts/opensearch/templates/statefulset-opensearch.yaml b/deploy/charts/opensearch/templates/statefulset-opensearch.yaml index 15dd3b280..855e87552 100644 --- a/deploy/charts/opensearch/templates/statefulset-opensearch.yaml +++ b/deploy/charts/opensearch/templates/statefulset-opensearch.yaml @@ -42,7 +42,7 @@ spec: {{- if gt (len $nodeCertFiles) 0 }} initContainers: - name: prepare-opensearch-certs - image: "{{ .Values.opensearch.image.repository }}:{{ .Values.opensearch.image.tag }}" + image: {{ include "cogstack-opensearch.image" .Values.opensearch.image | quote }} imagePullPolicy: {{ .Values.opensearch.image.pullPolicy }} securityContext: {{- toYaml .Values.opensearch.securityContext | nindent 12 }} @@ -78,7 +78,7 @@ spec: {{- end }} containers: - name: opensearch - image: "{{ .Values.opensearch.image.repository }}:{{ .Values.opensearch.image.tag }}" + image: {{ include "cogstack-opensearch.image" .Values.opensearch.image | quote }} imagePullPolicy: {{ .Values.opensearch.image.pullPolicy }} command: ["/bin/bash", "/opt/cogstack/search-entrypoint.sh"] args: ["opensearch"] diff --git a/deploy/charts/opensearch/values.yaml b/deploy/charts/opensearch/values.yaml index 0418b2dba..08aa9d4aa 100644 --- a/deploy/charts/opensearch/values.yaml +++ b/deploy/charts/opensearch/values.yaml @@ -52,6 +52,7 @@ opensearch: image: repository: opensearchproject/opensearch tag: "3.5.0" + digest: sha256:dbb01641baadae5104e18acd888bf05e8fdd9af3567fd30624a76ba3e5a31dec pullPolicy: IfNotPresent clusterName: elasticsearch-cogstack-cluster javaOpts: "-Xms512m -Xmx512m -Des.failure_store_feature_flag_enabled=true --enable-native-access=ALL-UNNAMED" @@ -128,6 +129,7 @@ dashboards: image: repository: opensearchproject/opensearch-dashboards tag: "3.5.0" + digest: sha256:03afc96f2f50a02246294aaa37adffc41270104676eff7c5f377c40ae9dd1ff2 pullPolicy: IfNotPresent serverName: cogstack-kibana service: diff --git a/deploy/charts/postgresql/values.yaml b/deploy/charts/postgresql/values.yaml index ea40088f0..3e5a73bf9 100644 --- a/deploy/charts/postgresql/values.yaml +++ b/deploy/charts/postgresql/values.yaml @@ -26,7 +26,7 @@ usersEnvFile: cluster: enabled: true instances: 3 - imageName: ghcr.io/cloudnative-pg/postgresql:18.1-system-trixie + imageName: ghcr.io/cloudnative-pg/postgresql:18.1-system-trixie@sha256:51bc94428be3df3e234447278e33803544a93d27a57a4b6af737f218d6472d12 imagePullPolicy: IfNotPresent databaseName: "" ownerName: "" diff --git a/deploy/database.env b/deploy/database.env index 3faeeaa5e..00052a62a 100644 --- a/deploy/database.env +++ b/deploy/database.env @@ -14,7 +14,7 @@ DATABASE_DOCKER_RAM=1g # ClickHouse analytics/ingestion database. These credentials are suitable for # local development only and must be overridden in shared or production use. -CLICKHOUSE_DOCKER_IMAGE=clickhouse/clickhouse-server:26.8 +CLICKHOUSE_DOCKER_IMAGE=clickhouse/clickhouse-server:26.8@sha256:d426d2f612e1e189c90f9a3f8dcd85a03c6851273833817a296bb327e63aa7a7 CLICKHOUSE_DB=cogstack CLICKHOUSE_USER=cogstack CLICKHOUSE_PASSWORD=clickhouse diff --git a/deploy/elasticsearch.env b/deploy/elasticsearch.env index ed9186166..9b9d4556d 100644 --- a/deploy/elasticsearch.env +++ b/deploy/elasticsearch.env @@ -12,7 +12,10 @@ ELASTICSEARCH_VERSION=opensearch # - opensearch : opensearchproject/opensearch:3.9.0 # the custom cogstack image is always based on the last image of ES native -ELASTICSEARCH_DOCKER_IMAGE=opensearchproject/opensearch:3.9.0 +ELASTICSEARCH_DOCKER_IMAGE=opensearchproject/opensearch:3.9.0@sha256:3b55f6b6249cfdb92b52ee59adc8a266203ba9ef9f5d82544fad6f1af2f3609e + +# Native Elasticsearch certificate utility; docker.elastic.co has no latest tag. +ES_NATIVE_CERTS_DOCKER_IMAGE=docker.elastic.co/elasticsearch/elasticsearch:8.19.6@sha256:bfb2108076735fbfc5dcd5d49f12f5aa93fab9759034c55f7da3b4345606c903 ELASTICSEARCH_LOG_LEVEL=INFO @@ -221,7 +224,7 @@ KIBANA_CONFIG_FILE_VERSION=opensearch_dashboards # - opensearch : opensearchproject/opensearch-dashboards:3.9.0 # the custom cogstack image is always based on the last image of ES native -ELASTICSEARCH_KIBANA_DOCKER_IMAGE=opensearchproject/opensearch-dashboards:3.9.0 +ELASTICSEARCH_KIBANA_DOCKER_IMAGE=opensearchproject/opensearch-dashboards:3.9.0@sha256:256d4cea5928868ac09bca642a0ac8285a2b7e4b7fdd45c4ec06e75a5cf3184c KIBANA_SERVER_NAME="cogstack-kibana" KIBANA_PUBLIC_BASE_URL="https://elasticsearch-1:5601" @@ -258,7 +261,7 @@ ELASTICSEARCH_XPACK_SECURITY_REPORTING_ENCRYPTION_KEY="e0Y1gTxHWOopIWMTtpjQsDS6K ######################################################################### METRICBEAT Env vars ########################################################################## -METRICBEAT_IMAGE="docker.elastic.co/beats/metricbeat:9.5.2" +METRICBEAT_IMAGE="docker.elastic.co/beats/metricbeat:9.5.2@sha256:aa7e217582804a2deb4a962e8795e92ea0723086fe04d3103d9e6583032615ec" METRICBEAT_DOCKER_SHM=512m METRICBEAT_DOCKER_CPU_MIN=1 @@ -275,7 +278,7 @@ FILEBEAT_STARTUP_COMMAND="-e --strict.perms=false" FILEBEAT_HOST="https://elasticsearch-1:9200" -FILEBEAT_IMAGE="docker.elastic.co/beats/filebeat:9.5.2" +FILEBEAT_IMAGE="docker.elastic.co/beats/filebeat:9.5.2@sha256:17aafbe925a43919b2eee539b1b0712e19aefd01247d4a1b855c91ca8e1ba2d6" FILEBEAT_DOCKER_SHM=512m diff --git a/deploy/general.env b/deploy/general.env index 843333f2e..bd27cd8ff 100644 --- a/deploy/general.env +++ b/deploy/general.env @@ -4,7 +4,7 @@ # set only CPU_ARCHITECTURE to amd/arm64 # remove if this causes issues on any other platform # possible values: amd64, arm64 -CPU_ARCHITECTURE=amd64 +CPU_ARCHITECTURE=arm64 DOCKER_DEFAULT_PLATFORM=linux/${CPU_ARCHITECTURE:-amd64} COMPOSE_BAKE=True diff --git a/deploy/gitea.env b/deploy/gitea.env index 6d312a6ae..8ac3523b0 100644 --- a/deploy/gitea.env +++ b/deploy/gitea.env @@ -25,7 +25,7 @@ GITEA_LOCAL_KEY_TITLE="gitea-cogstack-$(hostname)-$(date +%s)" GITEA_DEFAULT_MAIN_REMOTE_NAME="cogstack-gitea" GITEA_DOCKER_SHM_SIZE=512m -GITEA_DOCKER_IMAGE=gitea/gitea:1.27-rootless +GITEA_DOCKER_IMAGE=gitea/gitea:1.27-rootless@sha256:1c17ecaead42eb3b5391553d8708103a4beb0e86edf5b9ebc1eb269c318845f2 GITEA_DOCKER_CPU_MIN=1 GITEA_DOCKER_CPU_MAX=1 GITEA_DOCKER_RAM=1g diff --git a/deploy/nginx.env b/deploy/nginx.env index 5b52b10da..214092e51 100644 --- a/deploy/nginx.env +++ b/deploy/nginx.env @@ -1,3 +1,6 @@ +# Published nginx proxy image, pinned to its multi-platform registry manifest. +NIFI_NGINX_DOCKER_IMAGE=cogstacksystems/nifi-nginx@sha256:486c933b05e14b4a3a1e4f966d832b27b2097f6789d8c9fbeb0c129d97a7136f + NGINX_KIBANA_HOST=kibana NGINX_KIBANA_PROXY_PORT=5601 NGINX_ES_NODE_SOURCE_INSTANCE_NAME="elasticsearch-1" diff --git a/deploy/nifi.env b/deploy/nifi.env index 7afe3520e..ca229cee7 100644 --- a/deploy/nifi.env +++ b/deploy/nifi.env @@ -23,8 +23,12 @@ NIFI_DOCKER_LOG_NUM_FILES=10 # NiFi base image and extension version used by local image builds. NIFI_VERSION="2.12.0" -# NiFi Docker image -NIFI_DOCKER_IMAGE="cogstacksystems/cogstack-nifi:latest" +# Published NiFi 2.12.0 image, pinned to its multi-platform registry manifest. +NIFI_DOCKER_IMAGE="cogstacksystems/cogstack-nifi@sha256:f3f0ce9c23f9e503042f646675ddef372acda5cffc2491f07a38996a95a948dd" + +# Local builds use a separate tag; never tag a build with the published digest. +NIFI_LOCAL_DOCKER_IMAGE="cogstack/cogstack-nifi:local" +NIFI_BASE_IMAGE="apache/nifi:2.12.0@sha256:6f7f2840dbf178c6dd3fa13435b9b358f3be3c6fb7417de467a35a267f4d28ad" ############################################################################################################################## diff --git a/deploy/ollama.env b/deploy/ollama.env index 89eb8b02f..112f74379 100644 --- a/deploy/ollama.env +++ b/deploy/ollama.env @@ -1,7 +1,7 @@ ####################################################################### Ollama Env vars ####################################################################### -# Official Ollama server image. -OLLAMA_DOCKER_IMAGE=ollama/ollama:latest +# Official Ollama server image, pinned to the registry manifest resolved on 2026-10-29. +OLLAMA_DOCKER_IMAGE=ollama/ollama@sha256:b2f0122c476bc9fe7ce679d8bb4bd555351e25e2554eefb156f61cb0a89cf429 OLLAMA_CONTAINER_NAME=cogstack-ollama # OpenSearch's agent framework can serialize tool-call indexes as floats diff --git a/deploy/openwebui.env b/deploy/openwebui.env index 48f01ccbb..5d9fef842 100644 --- a/deploy/openwebui.env +++ b/deploy/openwebui.env @@ -1,8 +1,7 @@ ###################################################################### Open WebUI Env vars ###################################################################### -# Official Open WebUI image. Pin this tag to a numbered release when a -# reproducible production deployment is required. -OPENWEBUI_DOCKER_IMAGE=ghcr.io/open-webui/open-webui:main +# Official Open WebUI image, pinned to the registry manifest resolved on 2026-10-29. +OPENWEBUI_DOCKER_IMAGE=ghcr.io/open-webui/open-webui@sha256:096c8f3d3132016210ca87b0b4dff01d2d66d9fe4fee088fae13fc30f9ac54f3 OPENWEBUI_CONTAINER_NAME=cogstack-open-webui # Open WebUI reaches Ollama through the private Compose network. Do not use diff --git a/deploy/services.dev.yml b/deploy/services.dev.yml index f9b6f448e..94cf738ce 100644 --- a/deploy/services.dev.yml +++ b/deploy/services.dev.yml @@ -108,11 +108,14 @@ services: #---------------------------------------------------------------------------# nifi: <<: *nifi-common + image: ${NIFI_LOCAL_DOCKER_IMAGE:-cogstack/cogstack-nifi:local} + pull_policy: build build: context: .. dockerfile: nifi/Dockerfile args: NIFI_VERSION: ${NIFI_VERSION:-2.12.0} + NIFI_BASE_IMAGE: ${NIFI_BASE_IMAGE:?Source deploy/export_env_vars.sh first} container_name: cogstack-nifi hostname: nifi shm_size: ${NIFI_DOCKER_SHM_SIZE:-"1g"} @@ -140,6 +143,7 @@ services: logging: *nifi-logging-common nifi-nginx: + pull_policy: build build: context: ../services/nginx dockerfile: Dockerfile @@ -170,7 +174,7 @@ services: # Observability stack # #---------------------------------------------------------------------------# otel-collector: - image: ${OTEL_COLLECTOR_DOCKER_IMAGE:-otel/opentelemetry-collector-contrib:0.132.0} + image: ${OTEL_COLLECTOR_DOCKER_IMAGE:?Source deploy/export_env_vars.sh first} container_name: cogstack-otel-collector restart: unless-stopped command: ["--config=/etc/otelcol-contrib/config.yaml"] @@ -187,7 +191,7 @@ services: logging: *nifi-logging-common data-prepper: - image: ${DATA_PREPPER_DOCKER_IMAGE:-opensearchproject/data-prepper:2.11.0} + image: ${DATA_PREPPER_DOCKER_IMAGE:?Source deploy/export_env_vars.sh first} container_name: cogstack-data-prepper restart: unless-stopped env_file: *all-env diff --git a/deploy/services.yml b/deploy/services.yml index 194cf138e..9f5ae736a 100644 --- a/deploy/services.yml +++ b/deploy/services.yml @@ -158,7 +158,7 @@ x-es-common-volumes: &es-common-volumes x-es-common: &es-common <<: *common-ulimits - image: ${ELASTICSEARCH_DOCKER_IMAGE:-opensearchproject/opensearch:latest} + image: ${ELASTICSEARCH_DOCKER_IMAGE:?Source deploy/export_env_vars.sh first} entrypoint: ["/bin/bash", "/usr/local/bin/cogstack-search-entrypoint.sh"] environment: ELASTICSEARCH_VERSION: ${ELASTICSEARCH_VERSION:-opensearch} @@ -180,7 +180,7 @@ x-es-common: &es-common x-metricbeat-common: &metricbeat-common <<: *common-ulimits - image: ${METRICBEAT_IMAGE:-docker.elastic.co/beats/metricbeat:latest} + image: ${METRICBEAT_IMAGE:?Source deploy/export_env_vars.sh first} command: -e --strict.perms=false restart: unless-stopped shm_size: ${METRICBEAT_DOCKER_SHM:-1g} @@ -205,7 +205,7 @@ x-metricbeat-common: &metricbeat-common x-filebeat-common: &filebeat-common <<: *common-ulimits - image: ${FILEBEAT_IMAGE:-docker.elastic.co/beats/filebeat:latest} + image: ${FILEBEAT_IMAGE:?Source deploy/export_env_vars.sh first} command: ${FILEBEAT_STARTUP_COMMAND:-'-e --strict.perms=false'} restart: unless-stopped shm_size: ${FILEBEAT_DOCKER_SHM:-1g} @@ -238,7 +238,7 @@ services: #---------------------------------------------------------------------------# samples-db: <<: *db-common - image: postgres:18.6-trixie + image: postgres:18.6-trixie@sha256:979bccfde35445db618c65107d27d08c6c21339d8dc3ad4c830c6633947be0e4 container_name: cogstack-samples-db platform: linux/amd64 environment: @@ -265,7 +265,7 @@ services: #---------------------------------------------------------------------------# cogstack-databank-db: <<: *db-common - image: postgres:18.1-trixie + image: postgres:18.1-trixie@sha256:1090bc3a8ccfb0b55f78a494d76f8d603434f7e4553543d6e807bc7bd6bbd17f container_name: cogstack-production-databank-db platform: linux/amd64 environment: @@ -288,7 +288,7 @@ services: cogstack-databank-db-mssql: <<: *db-common - image: mcr.microsoft.com/mssql/server:2019-latest + image: mcr.microsoft.com/mssql/server:2019-latest@sha256:07d7dedecdbfcf6776035dc5eefb5bfd9aa1804b2f6b99f7dedb397d8a599069 container_name: cogstack-production-databank-db-mssql environment: - ACCEPT_EULA=y @@ -308,7 +308,7 @@ services: # ClickHouse analytics database # #---------------------------------------------------------------------------# clickhouse: - image: ${CLICKHOUSE_DOCKER_IMAGE:-clickhouse/clickhouse-server:26.8} + image: ${CLICKHOUSE_DOCKER_IMAGE:?Source deploy/export_env_vars.sh first} container_name: cogstack-clickhouse hostname: clickhouse restart: unless-stopped @@ -350,7 +350,7 @@ services: opensearch-performance-analyzer-init: # Named volumes previously mounted over metadata files are owned by root. # Prepare the database directories for OpenSearch's uid/gid without touching data. - image: ${ELASTICSEARCH_DOCKER_IMAGE:-opensearchproject/opensearch:latest} + image: ${ELASTICSEARCH_DOCKER_IMAGE:?Source deploy/export_env_vars.sh first} user: "0" entrypoint: ["/bin/sh", "-ec"] command: @@ -364,7 +364,7 @@ services: es_native_create_certs: container_name: es_create_certs - image: docker.elastic.co/elasticsearch/elasticsearch:latest + image: ${ES_NATIVE_CERTS_DOCKER_IMAGE:?Source deploy/export_env_vars.sh first} shm_size: ${ELASTICSEARCH_DOCKER_SHM_SIZE:-1g} env_file: *es-env restart: "no" @@ -513,7 +513,7 @@ services: #---------------------------------------------------------------------------# kibana: <<: *common-ulimits - image: ${ELASTICSEARCH_KIBANA_DOCKER_IMAGE:-opensearchproject/opensearch-dashboards:latest} + image: ${ELASTICSEARCH_KIBANA_DOCKER_IMAGE:?Source deploy/export_env_vars.sh first} container_name: cogstack-kibana shm_size: ${KIBANA_DOCKER_SHM_SIZE:-1g} restart: always @@ -551,7 +551,7 @@ services: # Ollama local model inference # #---------------------------------------------------------------------------# ollama: - image: ${OLLAMA_DOCKER_IMAGE:-ollama/ollama:latest} + image: ${OLLAMA_DOCKER_IMAGE:?Source deploy/export_env_vars.sh first} container_name: ${OLLAMA_CONTAINER_NAME:-cogstack-ollama} platform: ${DOCKER_DEFAULT_PLATFORM:-linux/amd64} restart: unless-stopped @@ -589,6 +589,7 @@ services: build: context: ../services/ollama-openai-proxy image: ${OLLAMA_OPENAI_PROXY_DOCKER_IMAGE:-cogstack/ollama-openai-proxy:local} + pull_policy: build container_name: ${OLLAMA_OPENAI_PROXY_CONTAINER_NAME:-cogstack-ollama-openai-proxy} restart: unless-stopped read_only: true @@ -619,7 +620,7 @@ services: # Open WebUI for local Ollama chat and model management # #---------------------------------------------------------------------------# open-webui: - image: ${OPENWEBUI_DOCKER_IMAGE:-ghcr.io/open-webui/open-webui:main} + image: ${OPENWEBUI_DOCKER_IMAGE:?Source deploy/export_env_vars.sh first} container_name: ${OPENWEBUI_CONTAINER_NAME:-cogstack-open-webui} platform: ${DOCKER_DEFAULT_PLATFORM:-linux/amd64} restart: unless-stopped @@ -652,7 +653,7 @@ services: #---------------------------------------------------------------------------# nifi: <<: *nifi-common - image: ${NIFI_DOCKER_IMAGE:-cogstacksystems/cogstack-nifi:latest} + image: ${NIFI_DOCKER_IMAGE:?Source deploy/export_env_vars.sh first} container_name: cogstack-nifi hostname: nifi shm_size: ${NIFI_DOCKER_SHM_SIZE:-"1g"} @@ -680,7 +681,7 @@ services: logging: *nifi-logging-common nifi-nginx: - image: cogstacksystems/nifi-nginx:latest + image: ${NIFI_NGINX_DOCKER_IMAGE:?Source deploy/export_env_vars.sh first} container_name: cogstack-nifi-nginx restart: always shm_size: ${NGINX_SHM_SIZE:-1g} @@ -710,7 +711,7 @@ services: gitea: <<: *common-ulimits container_name: cogstack-gitea - image: ${GITEA_DOCKER_IMAGE:-gitea/gitea:rootless-latest} + image: ${GITEA_DOCKER_IMAGE:?Source deploy/export_env_vars.sh first} shm_size: ${GITEA_DOCKER_SHM_SIZE:-"1g"} restart: always deploy: diff --git a/deploy/sglang.env b/deploy/sglang.env index f0a976cbe..94e300f0a 100644 --- a/deploy/sglang.env +++ b/deploy/sglang.env @@ -11,7 +11,7 @@ SGLANG_BASE_URL=${SGLANG_BASE_URL:-http://host.docker.internal:30000/v1} # Exact served model ID from GET /v1/models (or SGLang's --served-model-name). # Leave empty to discover it automatically when the server exposes one model. -SGLANG_MODEL=${SGLANG_MODEL:-"Qwen3.8-27B-4bit"} +SGLANG_MODEL=${SGLANG_MODEL:-"mlx-community/Qwen3.8-27B-4bit"} # Optional: match the server's --api-key. Empty means no Authorization header. # Prefer exporting the key in your shell instead of saving it in this file. diff --git a/deploy/telemetry.env b/deploy/telemetry.env index 4a3b93775..e59ec8aab 100644 --- a/deploy/telemetry.env +++ b/deploy/telemetry.env @@ -3,8 +3,8 @@ ############################################################################################################################## # Docker images -OTEL_COLLECTOR_DOCKER_IMAGE=otel/opentelemetry-collector-contrib:0.132.0 -DATA_PREPPER_DOCKER_IMAGE=opensearchproject/data-prepper:2.11.0 +OTEL_COLLECTOR_DOCKER_IMAGE=otel/opentelemetry-collector-contrib:0.132.0@sha256:e0f565815b2b8e78eb9fdbb80f6190c921ab323aaa8ccceab3255c6d4225f4af +DATA_PREPPER_DOCKER_IMAGE=opensearchproject/data-prepper:2.11.0@sha256:45d644fe7ec1724866d22e8c5cb5ddca9f963a4c5ecff872048e93c45de001df # Host ports exposed by observability services OTEL_COLLECTOR_OTLP_GRPC_PORT=4317 diff --git a/docs/deploy/air-gapped.md b/docs/deploy/air-gapped.md index 812d16c54..cbec1cf7b 100644 --- a/docs/deploy/air-gapped.md +++ b/docs/deploy/air-gapped.md @@ -58,7 +58,7 @@ assumes the repository has already been cloned into `cogstack_nifi`: ```bash cd cogstack_nifi git switch --detach -git submodule update --init --recursive +git submodule update --init --recursive --checkout git lfs pull git submodule foreach --recursive git lfs pull git status --short diff --git a/docs/deploy/deployment.md b/docs/deploy/deployment.md index 6875c6379..015df7809 100644 --- a/docs/deploy/deployment.md +++ b/docs/deploy/deployment.md @@ -261,7 +261,11 @@ make -C deploy remote-delete-service \ | `make init-security-elasticsearch` | Generate the native Elasticsearch certificate set if missing | | `make git-freeze-security` | Freeze all security submodules (read-only) | | `make git-unfreeze-security` | Unfreeze security submodules | -| `make git-update-submodules` | Update all submodules | +| `make git-init-submodules` | Initialize submodules at the recorded commits | +| `make git-update-submodules` | Explicitly upgrade direct submodules for review; no staging or commits | + +See [Repeatable deployments](./reproducible.md) for image pins, submodule +upgrades, and local build commands. --- diff --git a/docs/deploy/main.md b/docs/deploy/main.md index 1a42899ec..87cdfbd1e 100755 --- a/docs/deploy/main.md +++ b/docs/deploy/main.md @@ -49,7 +49,7 @@ Execute the following commands in the root directory of the repo: 1. `git-lfs pull` 2. (OPTIONAL, if you already have the software in [this section installed](#software-requirements-linuxmacos)) `bash ./scripts/installation_utils/install_docker_and_utils.sh`, and wait for it to finish. It may take a while to get all the packages, and it will prompt for `sudo` when needed. 3. `cd deploy` -4. `make git-update-submodules` +4. `make git-init-submodules` 5. `make init-security` 6. check that docker works correctly : `docker pull hello-world` 7. if no errors, run: `docker run --rm hello-world`, it should run without issues @@ -61,7 +61,7 @@ Execute the following commands in the root directory of the repo: !!! warning - Ensure all Git submodules are initialized and updated with `make git-update-submodules`. + Initialize Git submodules at the recorded commits with `make git-init-submodules`. !!! warning diff --git a/docs/deploy/reproducible.md b/docs/deploy/reproducible.md new file mode 100644 index 000000000..e6657cc91 --- /dev/null +++ b/docs/deploy/reproducible.md @@ -0,0 +1,132 @@ +# Repeatable deployments + +The core deployment records its software inputs in Git: submodule commits, +container image digests, and the application configuration. Use the same +repository revision and container platform to deploy those inputs again. +Existing volumes, generated certificates, credentials, and local configuration +changes remain part of each deployment's state. + +## Initialize a checkout + +Run these commands from the repository root: + +```bash +git lfs pull +make -C deploy git-init-submodules +make -C deploy init-security +make -C deploy start-data-infra +``` + +`git-init-submodules` checks out the exact commits recorded by the repository, +including nested submodules. It uses Git's explicit `--checkout` procedure and +does not select newer releases, stage files, or create commits. See the +[Git submodule documentation](https://git-scm.com/docs/git-submodule). + +CI checks these recorded commits with read-only repository permissions. It no +longer advances dependencies or pushes synchronization commits when a branch +changes. + +## Upgrade submodules explicitly + +In a development checkout, run: + +```bash +make -C deploy git-update-submodules +git diff --ignore-submodules=none --submodule=log +``` + +The upgrade command selects the highest version-sorted stable tag reachable +from each direct dependency's default branch. It falls back to other release +tags, then the default branch when no tags exist. Nested dependencies stay at +the commits recorded by the selected release. Local changes in initialized +submodules must be committed or stashed before upgrading. + +Review and test the resulting changes, then explicitly stage the submodule +paths and commit them in a pull request. The helper leaves the parent index and +commit history unchanged, including any unrelated staged work. + +The repository configures Git to hide routine submodule changes in ordinary +status output, so use `--ignore-submodules=none` when reviewing upgrades. + +## Container image pins + +Published images use `repository:version@sha256:...` or +`repository@sha256:...` references. The digest selects the image even when a +publisher later moves a tag. See Docker's +[image pinning guidance](https://docs.docker.com/build/building/best-practices/#pin-base-image-versions). + +Image selections live in: + +| Deployment | Image configuration | +|---|---| +| Core Compose services | `deploy/nifi.env`, `deploy/nginx.env`, `deploy/elasticsearch.env`, `deploy/database.env`, and `deploy/gitea.env` | +| Ollama and Open WebUI | `deploy/ollama.env` and `deploy/openwebui.env` | +| Sample PostgreSQL, databank PostgreSQL, and MSSQL | `deploy/services.yml` | +| Development observability services | `deploy/telemetry.env` | +| Owned Helm charts | `deploy/charts/*/values.yaml`; NiFi also reads `deploy/nifi.env` | + +Compose requires these image variables to be loaded. The Make targets load +them automatically. For direct Compose commands: + +```bash +cd deploy +source ./export_env_vars.sh +docker compose -f services.yml config --images +``` + +To update an image, inspect the chosen release without pulling its layers: + +```bash +docker buildx imagetools inspect repository/image:release --format '{{.Manifest.Digest}}' +``` + +Record the returned manifest digest alongside the version in the relevant +tracked configuration, test it, and submit the change for review. Use the +multi-platform index digest when both amd64 and arm64 are supported. + +For the NiFi and OpenSearch Helm charts, `image.digest` takes precedence over +`image.tag`. Update the digest together with the tag when selecting another +image. NiFi's imported `NIFI_DOCKER_IMAGE` remains the complete image reference +when supplied through its env file. + +## Local image builds + +Published image references and local build tags are separate: + +```bash +make -C deploy build-nifi-image +make -C deploy start-nifi-dev +``` + +Local NiFi builds use `NIFI_LOCAL_DOCKER_IMAGE`. Keep `NIFI_VERSION` and +`NIFI_BASE_IMAGE` aligned when updating NiFi, since the version also selects +its NAR extensions. The direct Docker build defaults live in `nifi/Dockerfile`. + +Services built from this checkout use `pull_policy: build` so Compose builds +the current source instead of reusing an unrelated cached local image. Their +base images are pinned in the Dockerfiles or supplied build arguments. + +NiFi source builds still resolve operating-system packages and external build +tools at build time. For repeatable production deployment, publish the tested +build and record its resulting image digest. + +## Validate the pins + +These checks render configuration and use temporary local Git repositories; +they do not start services or change this checkout's submodule commits: + +```bash +python3 -B scripts/tests/check_deployment_images.py --helm +python3 -B -m unittest discover -s scripts/tests -p test_deployment_images.py -v +python3 -B -m unittest discover -s scripts/tests -p test_submodule_checkout.py -v +``` + +The image check requires Docker Compose and Helm, but no running Docker daemon +or Kubernetes cluster. It validates both core Compose files, their local build +bases, and the three owned Helm charts. Helm image validation supplies an +existing credentials Secret name; deployment-specific credentials and TLS +Secrets must still be provided for an installation. + +Optional services such as OCR, MedCAT, and JupyterHub use deployment definitions +inside their pinned submodules. Image and model versioning within those +definitions is owned by the corresponding service repositories. diff --git a/docs/index.md b/docs/index.md index 7808e1135..9f43a6ff3 100644 --- a/docs/index.md +++ b/docs/index.md @@ -60,12 +60,16 @@ Navigate to the root folder of the repository ```bash # from repository root git lfs pull -make -C deploy git-update-submodules +make -C deploy git-init-submodules make -C deploy help make -C deploy init-security make -C deploy start-data-infra ``` +Setup uses the recorded submodule commits and pinned container images. See +[Repeatable deployments](deploy/reproducible.md) for explicit upgrades and +local image builds. + Once the services have started, you can find the following services at corresponding URLs: - NiFi: `https://localhost:8443` diff --git a/docs/nifi/main.md b/docs/nifi/main.md index 092a034c5..248e18cfd 100644 --- a/docs/nifi/main.md +++ b/docs/nifi/main.md @@ -41,15 +41,13 @@ For end-to-end scripting patterns (including `ExecuteStreamCommand` stdin/stdout For any deployment it is recommended to build and use the custom Docker image of Apache NiFi that will contain all the necessary configuration, drivers, custom user scripts and workflows. -The Docker image recipe is defined in `Dockerfile` file. -There are two images being built as part of CI process: +The Docker image recipe is defined in `nifi/Dockerfile`. Deployment uses the +published image digest recorded in `deploy/nifi.env`. CI also publishes branch, +commit, and release tags for selecting a future image update. -- `cogstacksystems/cogstack-nifi:latest` - the latest image built from `master` branch, -- `cogstacksystems/cogstack-nifi:dev-latest` - the latest image built from `devel` branch. - -There are also release images built for each release, for example: - -- `cogstacksystems/cogstack-nifi:1.0.0` - release 1.0 +`make -C deploy build-nifi-image` builds the source checkout using the separate +`NIFI_LOCAL_DOCKER_IMAGE` tag. Run it through `make -C deploy start-nifi-dev`. +See [Repeatable deployments](../deploy/reproducible.md) for the update procedure. ## Apache NiFi configuration diff --git a/mkdocs.yml b/mkdocs.yml index 577635861..b3a6c54af 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -44,6 +44,7 @@ nav: - 📋 Prerequisites: deploy/main.md - 📦 Air-gapped NiFi: deploy/air-gapped.md - 📦 Deployment: deploy/deployment.md + - 📌 Repeatable deployments: deploy/reproducible.md - ⚙️ Configuration: deploy/configuration.md - 🧩 Services: deploy/services.md - 🤖 OpenSearch AI with SGLang: deploy/sglang.md diff --git a/nifi/Dockerfile b/nifi/Dockerfile index 832551aaf..c95cc7360 100644 --- a/nifi/Dockerfile +++ b/nifi/Dockerfile @@ -1,10 +1,10 @@ -ARG NIFI_VERSION=2.12.0 +ARG NIFI_BASE_IMAGE=apache/nifi:2.12.0@sha256:6f7f2840dbf178c6dd3fa13435b9b358f3be3c6fb7417de467a35a267f4d28ad -FROM apache/nifi:${NIFI_VERSION} +FROM ${NIFI_BASE_IMAGE} SHELL ["/bin/bash", "-o", "pipefail", "-c"] -ARG NIFI_VERSION +ARG NIFI_VERSION=2.12.0 ARG GROOVY_VERSION=5.1.0 ARG HTTP_PROXY="" diff --git a/scripts/git_init_submodules_in_repo.sh b/scripts/git_init_submodules_in_repo.sh new file mode 100755 index 000000000..104ff1163 --- /dev/null +++ b/scripts/git_init_submodules_in_repo.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +# Restore exactly the submodule commits recorded by this checkout. +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +git -C "$repo_root" submodule update --init --recursive --checkout +echo "Submodules initialized at the recorded commits." diff --git a/scripts/git_update_submodules_in_repo.sh b/scripts/git_update_submodules_in_repo.sh index f327a4218..17d03ad13 100755 --- a/scripts/git_update_submodules_in_repo.sh +++ b/scripts/git_update_submodules_in_repo.sh @@ -1,16 +1,27 @@ #!/usr/bin/env bash set -euo pipefail -repo_root="$(git rev-parse --show-toplevel)" +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "$repo_root" -git submodule update --init --recursive +# Check before initialization so local edits are never carried into an upgrade. +# shellcheck disable=SC2016 +git submodule foreach --recursive ' + if [ -n "$(git status --porcelain --untracked-files=normal --ignore-submodules=none)" ]; then + echo "ERROR: $displaypath has local changes; commit or stash them before upgrading." >&2 + exit 1 + fi +' + +"$repo_root/scripts/git_init_submodules_in_repo.sh" echo "Updating submodules to latest release tag on each origin default branch (fallback: branch HEAD)..." # Variables in this block are intentionally expanded by each submodule shell. # shellcheck disable=SC2016 -git submodule foreach --recursive ' +# Upgrade only direct dependencies. Nested submodules must stay at the commits +# recorded by each selected dependency release. +git submodule foreach ' set -e # Fetch full branch refs + tag refs so tag selection is reliable. @@ -31,21 +42,21 @@ git submodule foreach --recursive ' latest="" if [ -n "$default_branch" ] && git show-ref --verify --quiet "refs/remotes/origin/$default_branch"; then - latest=$(git for-each-ref --merged "refs/remotes/origin/$default_branch" --sort=-creatordate --format="%(refname:strip=2)" refs/tags \ + latest=$(git for-each-ref --merged "refs/remotes/origin/$default_branch" --sort=-version:refname --format="%(refname:strip=2)" refs/tags \ | grep -Eiv "(^|[._/-])(alpha|beta|rc|pre|preview|snapshot)([0-9._-]*|$)" | head -n1 || true) if [ -z "$latest" ]; then - latest=$(git for-each-ref --merged "refs/remotes/origin/$default_branch" --sort=-creatordate --format="%(refname:strip=2)" refs/tags | head -n1 || true) + latest=$(git for-each-ref --merged "refs/remotes/origin/$default_branch" --sort=-version:refname --format="%(refname:strip=2)" refs/tags | head -n1 || true) fi fi # Fallback for repositories where tags are not merged into the default branch. if [ -z "$latest" ]; then - latest=$(git for-each-ref --sort=-creatordate --format="%(refname:strip=2)" refs/tags \ + latest=$(git for-each-ref --sort=-version:refname --format="%(refname:strip=2)" refs/tags \ | grep -Eiv "(^|[._/-])(alpha|beta|rc|pre|preview|snapshot)([0-9._-]*|$)" | head -n1 || true) fi if [ -z "$latest" ]; then - latest=$(git for-each-ref --sort=-creatordate --format="%(refname:strip=2)" refs/tags | head -n1 || true) + latest=$(git for-each-ref --sort=-version:refname --format="%(refname:strip=2)" refs/tags | head -n1 || true) fi if [ -n "$latest" ]; then @@ -57,27 +68,10 @@ git submodule foreach --recursive ' else echo "-> $name: no tags/branches found; keeping current commit" fi -' - -#git submodule foreach git pull origin main -submodule_paths=() -while IFS= read -r submodule_path; do - [[ -n "$submodule_path" ]] && submodule_paths+=("$submodule_path") -done < <(git config -f .gitmodules --get-regexp '^submodule\..*\.path$' | awk '{print $2}') - -if (( ${#submodule_paths[@]} > 0 )); then - git add -- "${submodule_paths[@]}" || true -fi -git commit -m "Update submodules to latest release tags (or default branch)" || echo "ℹ️ No changes to commit." -echo "✅ Submodule update complete." -# fix jupyter-hub cookie file permissions -for f in \ - "../services/cogstack-platform/apps/cogstack-jupyter-hub/config/jupyterhub_cookie_secret" \ - "services/cogstack-platform/apps/cogstack-jupyter-hub/config/jupyterhub_cookie_secret" -do - - [ -f "$f" ] && chmod 500 "$f" && echo "Fixing jupyter-hub cookie secret file permissions. $f" -done + git submodule update --init --recursive --checkout +' -exec "$@" +echo "Submodule upgrades are ready for review; nothing was staged or committed." +git diff --ignore-submodules=none --submodule=log +echo "After testing, stage the changed submodule paths and commit them explicitly." diff --git a/scripts/tests/check_deployment_images.py b/scripts/tests/check_deployment_images.py new file mode 100755 index 000000000..d75a66369 --- /dev/null +++ b/scripts/tests/check_deployment_images.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 +"""Validate resolved deployment images without starting Docker or Kubernetes.""" + +from __future__ import annotations + +import argparse +import json +import re +import shlex +import subprocess +import sys +from pathlib import Path +from string import Template + +ROOT = Path(__file__).resolve().parents[2] +PINNED_IMAGE = re.compile(r"[^\s@]+@sha256:[0-9a-f]{64}\Z") + + +def build_errors(build: dict, label: str) -> list[str]: + context = Path(build["context"]) + dockerfile = context / build.get("dockerfile", "Dockerfile") + arguments = {key: str(value) for key, value in build.get("args", {}).items() if value is not None} + stages: set[str] = set() + errors = [] + for line in dockerfile.read_text().splitlines(): + instruction, _, value = line.strip().partition(" ") + if instruction.upper() == "ARG": + key, separator, default = value.partition("=") + if separator: + arguments.setdefault(key, default.strip('"\'')) + elif instruction.upper() == "FROM": + words = shlex.split(value) + words = [word for word in words if not word.startswith("--")] + image = Template(words[0]).safe_substitute(arguments) + if image not in stages and image != "scratch" and not PINNED_IMAGE.fullmatch(image): + errors.append(f"{label}: unpinned build base {image}") + if len(words) == 3 and words[1].upper() == "AS": + stages.add(words[2]) + return errors + + +def compose_errors(config: dict, label: str) -> list[str]: + errors = [] + for name, service in config["services"].items(): + service_label = f"{label}/{name}" + if "build" in service: + if service.get("pull_policy") != "build": + errors.append(f"{service_label}: local builds require pull_policy: build") + errors.extend(build_errors(service["build"], service_label)) + elif not PINNED_IMAGE.fullmatch(service.get("image", "")): + errors.append(f"{service_label}: image must include an immutable @sha256 digest") + return errors + + +def render_compose(path: Path) -> dict: + result = subprocess.run( + [ + "bash", "-c", + 'source "$1" >/dev/null; set -euo pipefail; ' + 'exec docker compose --profile "*" -f "$2" config --format json', + "check-deployment-images", str(ROOT / "deploy/export_env_vars.sh"), str(path), + ], + text=True, capture_output=True, check=True, timeout=60, + ) + return json.loads(result.stdout) + + +def helm_errors() -> list[str]: + errors = [] + for chart in ("nifi", "opensearch", "postgresql"): + command = [ + "helm", "template", f"check-{chart}", str(ROOT / "deploy/charts" / chart), + "-f", str(ROOT / "deploy/helm" / f"{chart}.values.yaml"), + ] + if chart == "opensearch": + # Image rendering only: deployment credentials are supplied externally. + command.extend([ + "--set", "credentials.create=false", "--set", "credentials.existingSecret=check-credentials", + ]) + result = subprocess.run(command, text=True, capture_output=True, check=True, timeout=60) + images = re.findall(r"^\s*(?:image|imageName):\s*(\S+)", result.stdout, re.MULTILINE) + if not images: + errors.append(f"Helm {chart}: no images rendered") + for image in images: + if not PINNED_IMAGE.fullmatch(image.strip('"\'')): + errors.append(f"Helm {chart}: unpinned image {image}") + return errors + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--helm", action="store_true", help="also render and check the owned Helm charts") + args = parser.parse_args() + errors = [] + try: + for name in ("services.yml", "services.dev.yml"): + errors.extend(compose_errors(render_compose(ROOT / "deploy" / name), name)) + if args.helm: + errors.extend(helm_errors()) + except (OSError, subprocess.SubprocessError, ValueError) as exc: + print(f"Deployment image validation failed: {exc}", file=sys.stderr) + if isinstance(exc, subprocess.CalledProcessError): + print(exc.stderr, file=sys.stderr) + return 1 + if errors: + print("Deployment image validation failed:", file=sys.stderr) + for error in errors: + print(f"- {error}", file=sys.stderr) + return 1 + print("Deployment images and local build bases are pinned; local services rebuild from source.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/tests/test_deployment_images.py b/scripts/tests/test_deployment_images.py new file mode 100755 index 000000000..129406e2d --- /dev/null +++ b/scripts/tests/test_deployment_images.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +"""Regression checks for image drift and stale local build reuse.""" + +from __future__ import annotations + +import tempfile +import unittest +from pathlib import Path + +from check_deployment_images import compose_errors + + +class DeploymentImagesTest(unittest.TestCase): + def test_numbered_tags_still_require_a_digest(self) -> None: + for image in ("example/app:latest", "example/app:1.2.3", "example/app@sha256:short"): + with self.subTest(image=image): + self.assertTrue(compose_errors({"services": {"app": {"image": image}}}, "test")) + image = "example/app:1.2.3@sha256:" + "a" * 64 + self.assertEqual(compose_errors({"services": {"app": {"image": image}}}, "test"), []) + + def test_local_builds_need_pinned_bases_and_source_rebuilds(self) -> None: + with tempfile.TemporaryDirectory() as directory: + dockerfile = Path(directory) / "Dockerfile" + dockerfile.write_text("FROM python:3.12-alpine\n") + service = {"build": {"context": directory}, "image": "example/app:local"} + errors = compose_errors({"services": {"app": service}}, "test") + self.assertEqual(len(errors), 2) + service["pull_policy"] = "build" + dockerfile.write_text("FROM python:3.12-alpine@sha256:" + "a" * 64 + "\n") + self.assertEqual(compose_errors({"services": {"app": service}}, "test"), []) + + def test_build_argument_overrides_are_validated(self) -> None: + with tempfile.TemporaryDirectory() as directory: + dockerfile = Path(directory) / "Dockerfile" + dockerfile.write_text("ARG BASE=python@sha256:" + "a" * 64 + "\nFROM ${BASE}\n") + service = { + "build": {"context": directory, "args": {"BASE": "python:latest"}}, + "pull_policy": "build", + } + self.assertTrue(compose_errors({"services": {"app": service}}, "test")) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tests/test_submodule_checkout.py b/scripts/tests/test_submodule_checkout.py new file mode 100755 index 000000000..f3bea4af7 --- /dev/null +++ b/scripts/tests/test_submodule_checkout.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 +"""Test submodule setup against local repositories without changing this checkout.""" + +from __future__ import annotations + +import os +import shutil +import subprocess +import tempfile +import unittest +from pathlib import Path + +PROJECT_DIR = Path(__file__).resolve().parents[2] + + +class SubmoduleCheckoutTest(unittest.TestCase): + def setUp(self) -> None: + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.env = { + **os.environ, + "GIT_CONFIG_GLOBAL": os.devnull, + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_ALLOW_PROTOCOL": "file", + "GIT_TERMINAL_PROMPT": "0", + } + self.nested = self.new_repo("nested-origin") + self.nested_pin = self.commit(self.nested, "old nested code") + self.dependency = self.new_repo("dependency-origin") + self.git(self.dependency, "submodule", "add", str(self.nested), "nested") + self.dependency_pin = self.commit(self.dependency, "old dependency code") + self.git(self.dependency, "tag", "v1.0.0") + + source = self.new_repo("source") + (source / "scripts").mkdir() + for name in ("git_init_submodules_in_repo.sh", "git_update_submodules_in_repo.sh"): + shutil.copy2(PROJECT_DIR / "scripts" / name, source / "scripts" / name) + self.git(source, "submodule", "add", str(self.dependency), "dependency") + self.git(source, "config", "-f", ".gitmodules", "submodule.dependency.ignore", "all") + self.commit(source, "parent code") + + # Both remotes advance after the parent records its dependency commits. + self.commit(self.nested, "new nested code") + self.git(self.nested, "tag", "v2.0.0") + self.dependency_release = self.commit(self.dependency, "released dependency code") + self.git(self.dependency, "tag", "v2.0.0") + self.commit(self.dependency, "unreleased dependency code") + self.git(self.dependency, "tag", "v3.0.0-rc1") + + self.checkout = self.root / "checkout" + self.git(self.root, "clone", "--quiet", str(source), str(self.checkout)) + self.parent_head = self.git(self.checkout, "rev-parse", "HEAD") + # An unrelated staged edit must never be included in an automatic commit. + (self.checkout / "payload.txt").write_text("staged parent edit\n") + self.git(self.checkout, "add", "payload.txt") + self.parent_index = self.git(self.checkout, "write-tree") + + def git(self, cwd: Path, *args: str) -> str: + result = subprocess.run( + ["git", *args], cwd=cwd, env=self.env, text=True, + capture_output=True, check=True, timeout=30, + ) + return result.stdout.strip() + + def new_repo(self, name: str) -> Path: + repo = self.root / name + repo.mkdir() + self.git(repo, "init", "--quiet", "--initial-branch=main") + self.git(repo, "config", "user.name", "Test") + self.git(repo, "config", "user.email", "test@example.invalid") + return repo + + def commit(self, repo: Path, text: str) -> str: + (repo / "payload.txt").write_text(text + "\n") + self.git(repo, "add", ".") + self.git(repo, "commit", "--quiet", "-m", text) + return self.git(repo, "rev-parse", "HEAD") + + def helper(self, action: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["bash", str(self.checkout / "scripts" / f"git_{action}_submodules_in_repo.sh")], + cwd=self.root, env=self.env, text=True, capture_output=True, timeout=30, + ) + + def assert_parent_untouched(self) -> None: + self.assertEqual(self.git(self.checkout, "rev-parse", "HEAD"), self.parent_head) + self.assertEqual(self.git(self.checkout, "write-tree"), self.parent_index) + + def test_initialization_uses_recorded_commits_and_is_repeatable(self) -> None: + # Explicit --checkout must override a contributor's merge preference. + self.git(self.checkout, "config", "submodule.dependency.update", "merge") + for _ in range(2): + result = self.helper("init") + self.assertEqual(result.returncode, 0, result.stderr) + dependency = self.checkout / "dependency" + self.assertEqual(self.git(dependency, "rev-parse", "HEAD"), self.dependency_pin) + self.assertEqual(self.git(dependency / "nested", "rev-parse", "HEAD"), self.nested_pin) + self.assert_parent_untouched() + + def test_upgrade_selects_release_and_keeps_nested_dependencies_pinned(self) -> None: + result = self.helper("update") + self.assertEqual(result.returncode, 0, result.stderr) + dependency = self.checkout / "dependency" + self.assertEqual(self.git(dependency, "rev-parse", "HEAD"), self.dependency_release) + self.assertEqual(self.git(dependency / "nested", "rev-parse", "HEAD"), self.nested_pin) + self.assert_parent_untouched() + + def test_upgrade_refuses_local_changes_in_nested_submodule(self) -> None: + result = self.helper("init") + self.assertEqual(result.returncode, 0, result.stderr) + dependency = self.checkout / "dependency" + (dependency / "nested" / "payload.txt").write_text("local edit\n") + result = self.helper("update") + self.assertNotEqual(result.returncode, 0) + self.assertIn("local changes", result.stderr) + self.assertEqual(self.git(dependency, "rev-parse", "HEAD"), self.dependency_pin) + self.assertEqual((dependency / "nested" / "payload.txt").read_text(), "local edit\n") + self.assert_parent_untouched() + + +if __name__ == "__main__": + unittest.main() diff --git a/security/scripts/create_es_native_certs.sh b/security/scripts/create_es_native_certs.sh index 8fee1b813..9270712c3 100755 --- a/security/scripts/create_es_native_certs.sh +++ b/security/scripts/create_es_native_certs.sh @@ -26,6 +26,11 @@ set -euo pipefail +# Compose requires the pinned image references as well as certificate settings. +# shellcheck disable=SC1091 +source "$(dirname "${BASH_SOURCE[0]}")/../../deploy/export_env_vars.sh" +set -euo pipefail + cleanup_es_native_artifacts() { docker ps -a -q --filter name="deploy-es_native_create_certs-run-*" | while IFS= read -r container_id; do diff --git a/services/nginx/Dockerfile b/services/nginx/Dockerfile index e92421d43..162b08d59 100644 --- a/services/nginx/Dockerfile +++ b/services/nginx/Dockerfile @@ -1,4 +1,4 @@ -FROM nginx:latest +FROM nginx@sha256:600f7cd45e27aa13e9304c834ab7c1b0ff39df85ee1dcc5087955455d77bc83b # Add configuration RUN rm /etc/nginx/conf.d/default.conf diff --git a/services/ollama-openai-proxy/Dockerfile b/services/ollama-openai-proxy/Dockerfile index 27f0c48fa..3a3086da8 100644 --- a/services/ollama-openai-proxy/Dockerfile +++ b/services/ollama-openai-proxy/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.12-alpine +FROM python:3.12-alpine@sha256:5789aa194f757888f41fa3d48e14b6feaa1b57c0af0b56fcb1372d6474d3bff0 ENV PYTHONDONTWRITEBYTECODE=1 \ PYTHONUNBUFFERED=1 From 577c0de42225c1a88b5a55b1a8c7c4cd896d2de8 Mon Sep 17 00:00:00 2001 From: vladd-bit Date: Thu, 29 Oct 2026 13:09:37 +0000 Subject: [PATCH 11/28] Consitency update for env files across docker containers. --- .github/workflows/compose_config.yml | 25 ++- .gitignore | 1 + README.md | 1 + deploy/Makefile | 58 ++++-- deploy/charts/nifi/templates/_helpers.tpl | 17 ++ deploy/charts/nifi/templates/statefulset.yaml | 14 ++ deploy/charts/nifi/values.yaml | 4 +- deploy/config_files.sh | 39 ++++ deploy/export_env_vars.sh | 143 +++++++------ deploy/general.env | 2 +- deploy/nifi.env | 2 +- deploy/services.dev.yml | 50 ++--- deploy/services.yml | 64 +++--- docs/deploy/configuration.md | 113 ++++++++++- docs/deploy/deployment.md | 2 + docs/deploy/reproducible.md | 1 + docs/index.md | 1 + docs/nifi/main.md | 10 +- scripts/opensearch-ai-init.sh | 7 +- scripts/tests/check_nifi_resources.py | 59 ++++++ scripts/tests/test_configuration.py | 188 ++++++++++++++++++ scripts/tests/test_opensearch_ai_init.py | 8 +- security/scripts/create_es_native_certs.sh | 2 - .../scripts/create_es_native_credentials.sh | 13 +- security/scripts/create_gitea_certs.sh | 7 +- security/scripts/create_keystore.sh | 7 +- security/scripts/create_nifi_certs.sh | 8 +- .../create_opensearch_client_admin_certs.sh | 10 +- .../create_opensearch_internal_passwords.sh | 13 +- .../scripts/create_opensearch_node_cert.sh | 13 +- security/scripts/create_opensearch_users.sh | 13 +- security/scripts/create_root_ca_cert.sh | 6 +- security/scripts/load_config.sh | 7 + security/scripts/update_opensearch_users.sh | 20 +- 34 files changed, 735 insertions(+), 193 deletions(-) create mode 100755 deploy/config_files.sh create mode 100755 scripts/tests/check_nifi_resources.py create mode 100755 scripts/tests/test_configuration.py create mode 100755 security/scripts/load_config.sh diff --git a/.github/workflows/compose_config.yml b/.github/workflows/compose_config.yml index 2463afef5..dfcc31230 100644 --- a/.github/workflows/compose_config.yml +++ b/.github/workflows/compose_config.yml @@ -6,6 +6,13 @@ on: paths: - "deploy/*.env" - "deploy/export_env_vars.sh" + - "deploy/config_files.sh" + - "deploy/Makefile" + - "security/scripts/**" + - "scripts/opensearch-ai-init.sh" + - "scripts/tests/check_nifi_resources.py" + - "scripts/tests/test_configuration.py" + - "scripts/tests/test_opensearch_ai_init.py" - "deploy/services*.yml" - "deploy/charts/**" - "deploy/helm/**" @@ -20,6 +27,13 @@ on: paths: - "deploy/*.env" - "deploy/export_env_vars.sh" + - "deploy/config_files.sh" + - "deploy/Makefile" + - "security/scripts/**" + - "scripts/opensearch-ai-init.sh" + - "scripts/tests/check_nifi_resources.py" + - "scripts/tests/test_configuration.py" + - "scripts/tests/test_opensearch_ai_init.py" - "deploy/services*.yml" - "deploy/charts/**" - "deploy/helm/**" @@ -54,13 +68,16 @@ jobs: - name: Validate Compose files shell: bash run: | - set -euo pipefail - source deploy/export_env_vars.sh - docker compose -f deploy/services.yml config --quiet - docker compose -f deploy/services.dev.yml config --quiet + make -C deploy check-config - name: Verify deployment image pins run: python3 -B scripts/tests/check_deployment_images.py --helm - name: Test image drift checks run: python3 -B -m unittest discover -s scripts/tests -p test_deployment_images.py -v + + - name: Test configuration profiles and resource guards + run: python3 -B -m unittest discover -s scripts/tests -p test_configuration.py -v + + - name: Test OpenSearch AI configuration + run: python3 -B -m unittest discover -s scripts/tests -p test_opensearch_ai_init.py -v diff --git a/.gitignore b/.gitignore index 5cb535b82..9b73edfe2 100644 --- a/.gitignore +++ b/.gitignore @@ -33,6 +33,7 @@ build # env files with passwords or local paths *.env +deploy/local/ security/certificates/ security/certificates/** security/env/ diff --git a/README.md b/README.md index 99f39ee8e..9ab211bcc 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,7 @@ This table describes repository layout. For setup and operations, use the deploy git lfs pull make -C deploy git-init-submodules make -C deploy help +make -C deploy check-config make -C deploy init-security make -C deploy start-data-infra ``` diff --git a/deploy/Makefile b/deploy/Makefile index c4ab2dd21..1e420825f 100644 --- a/deploy/Makefile +++ b/deploy/Makefile @@ -39,9 +39,11 @@ SECURITY_SCRIPTS_DIR ?= $(SECURITY_DIR)/scripts SECURITY_CERTIFICATES_DIR ?= $(SECURITY_DIR)/certificates SECURITY_ENV_DIR ?= $(SECURITY_DIR)/env OPENSEARCH_CERT_NAMES ?= +COGSTACK_CONFIG_DIR ?= +export COGSTACK_CONFIG_DIR define WITH_ENV -set -a && source ./export_env_vars.sh; +source ./export_env_vars.sh || exit $$?; endef # list all make targets in this file @@ -107,7 +109,8 @@ show-env: ## Print sorted environment variables after loading export_env_vars.sh init-security-root-ca: ## Generate the shared root CA only when no CA material exists @set -euo pipefail; \ - source "$(SECURITY_ENV_DIR)/certificates_general.env"; \ + $(WITH_ENV) \ + source "$${COGSTACK_CERTIFICATES_GENERAL_ENV_FILE}"; \ root_dir="$(SECURITY_CERTIFICATES_DIR)/root"; \ if [[ -f "$$root_dir/$${ROOT_CERTIFICATE_NAME}.key" \ && -f "$$root_dir/$${ROOT_CERTIFICATE_NAME}.pem" \ @@ -152,7 +155,8 @@ init-security-gitea: init-security-root-ca ## Generate a dedicated Gitea leaf ce init-security-opensearch: init-security-root-ca ## Generate OpenSearch node, client, and admin certificates when missing @set -euo pipefail; \ - source "$(SECURITY_ENV_DIR)/certificates_elasticsearch.env"; \ + $(WITH_ENV) \ + source "$${COGSTACK_CERTIFICATES_ELASTICSEARCH_ENV_FILE}"; \ if [[ -n "$(strip $(OPENSEARCH_CERT_NAMES))" ]]; then \ read -r -a node_names <<< "$(strip $(OPENSEARCH_CERT_NAMES))"; \ else \ @@ -191,7 +195,8 @@ init-security-opensearch: init-security-root-ca ## Generate OpenSearch node, cli init-security-elasticsearch: ## Generate native Elasticsearch certificates when the output directory is absent @set -euo pipefail; \ - source "$(SECURITY_ENV_DIR)/certificates_elasticsearch.env"; \ + $(WITH_ENV) \ + source "$${COGSTACK_CERTIFICATES_ELASTICSEARCH_ENV_FILE}"; \ elasticsearch_dir="$(SECURITY_CERTIFICATES_DIR)/elastic/elasticsearch"; \ certificate_set_complete=1; \ for required_file in \ @@ -221,7 +226,8 @@ init-security-elasticsearch: ## Generate native Elasticsearch certificates when init-security-search: ## Generate certificates for ELASTICSEARCH_VERSION from elasticsearch.env @set -euo pipefail; \ - source ./elasticsearch.env; \ + $(WITH_ENV) \ + source "$${COGSTACK_ELASTICSEARCH_ENV_FILE}"; \ case "$${ELASTICSEARCH_VERSION}" in \ opensearch|elasticsearch) \ $(MAKE) --no-print-directory "init-security-$${ELASTICSEARCH_VERSION}" \ @@ -240,25 +246,41 @@ init-security: ## Generate missing NiFi, Gitea, and selected search-backend cert .PHONY: init-security init-security-root-ca init-security-nifi init-security-gitea init-security-opensearch init-security-elasticsearch init-security-search helm-template-opensearch: ## Render OpenSearch chart using chart defaults plus ./helm/opensearch.values.yaml - helm template $(HELM_OPENSEARCH_RELEASE) $(HELM_OPENSEARCH_CHART) $(HELM_OPENSEARCH_VALUES_ARG) + $(WITH_ENV) helm template $(HELM_OPENSEARCH_RELEASE) $(HELM_OPENSEARCH_CHART) $(HELM_OPENSEARCH_VALUES_ARG) \ + --set-file envFile.raw="$${COGSTACK_ELASTICSEARCH_ENV_FILE}" \ + --set-file certificatesEnvFile.raw="$${COGSTACK_CERTIFICATES_ELASTICSEARCH_ENV_FILE}" \ + --set-file usersEnvFile.raw="$${COGSTACK_USERS_ELASTICSEARCH_ENV_FILE}" helm-install-opensearch: ## Install/upgrade OpenSearch chart using chart defaults plus ./helm/opensearch.values.yaml - helm upgrade --install $(HELM_OPENSEARCH_RELEASE) $(HELM_OPENSEARCH_CHART) $(HELM_OPENSEARCH_VALUES_ARG) --namespace $(HELM_OPENSEARCH_NAMESPACE) --create-namespace + $(WITH_ENV) helm upgrade --install $(HELM_OPENSEARCH_RELEASE) $(HELM_OPENSEARCH_CHART) $(HELM_OPENSEARCH_VALUES_ARG) --namespace $(HELM_OPENSEARCH_NAMESPACE) --create-namespace \ + --set-file envFile.raw="$${COGSTACK_ELASTICSEARCH_ENV_FILE}" \ + --set-file certificatesEnvFile.raw="$${COGSTACK_CERTIFICATES_ELASTICSEARCH_ENV_FILE}" \ + --set-file usersEnvFile.raw="$${COGSTACK_USERS_ELASTICSEARCH_ENV_FILE}" kube-install-cnpg-operator: ## Install CloudNativePG operator/CRDs from the pinned upstream manifest kubectl apply --server-side -f $(CNPG_OPERATOR_MANIFEST) helm-template-nifi: ## Render CogStack NiFi chart using chart defaults plus ./helm/nifi.values.yaml - helm template $(HELM_NIFI_RELEASE) $(HELM_NIFI_CHART) $(HELM_NIFI_VALUES_ARG) --namespace $(HELM_NIFI_NAMESPACE) + $(WITH_ENV) helm template $(HELM_NIFI_RELEASE) $(HELM_NIFI_CHART) $(HELM_NIFI_VALUES_ARG) --namespace $(HELM_NIFI_NAMESPACE) \ + --set-file envFile.raw="$${COGSTACK_NIFI_ENV_FILE}" \ + --set-file certificatesEnvFile.raw="$${COGSTACK_CERTIFICATES_NIFI_ENV_FILE}" \ + --set-file usersEnvFile.raw="$${COGSTACK_USERS_NIFI_ENV_FILE}" helm-install-nifi: ## Install/upgrade CogStack NiFi chart using chart defaults plus ./helm/nifi.values.yaml - helm upgrade --install $(HELM_NIFI_RELEASE) $(HELM_NIFI_CHART) $(HELM_NIFI_VALUES_ARG) --namespace $(HELM_NIFI_NAMESPACE) --create-namespace + $(WITH_ENV) helm upgrade --install $(HELM_NIFI_RELEASE) $(HELM_NIFI_CHART) $(HELM_NIFI_VALUES_ARG) --namespace $(HELM_NIFI_NAMESPACE) --create-namespace \ + --set-file envFile.raw="$${COGSTACK_NIFI_ENV_FILE}" \ + --set-file certificatesEnvFile.raw="$${COGSTACK_CERTIFICATES_NIFI_ENV_FILE}" \ + --set-file usersEnvFile.raw="$${COGSTACK_USERS_NIFI_ENV_FILE}" helm-template-postgresql: ## Render CloudNativePG PostgreSQL chart using chart defaults plus ./helm/postgresql.values.yaml - helm template $(HELM_POSTGRESQL_RELEASE) $(HELM_POSTGRESQL_CHART) $(HELM_POSTGRESQL_VALUES_ARG) --namespace $(HELM_POSTGRESQL_NAMESPACE) + $(WITH_ENV) helm template $(HELM_POSTGRESQL_RELEASE) $(HELM_POSTGRESQL_CHART) $(HELM_POSTGRESQL_VALUES_ARG) --namespace $(HELM_POSTGRESQL_NAMESPACE) \ + --set-file envFile.raw="$${COGSTACK_DATABASE_ENV_FILE}" \ + --set-file usersEnvFile.raw="$${COGSTACK_USERS_DATABASE_ENV_FILE}" helm-install-postgresql: ## Install/upgrade CloudNativePG PostgreSQL chart using chart defaults plus ./helm/postgresql.values.yaml - helm upgrade --install $(HELM_POSTGRESQL_RELEASE) $(HELM_POSTGRESQL_CHART) $(HELM_POSTGRESQL_VALUES_ARG) --namespace $(HELM_POSTGRESQL_NAMESPACE) --create-namespace + $(WITH_ENV) helm upgrade --install $(HELM_POSTGRESQL_RELEASE) $(HELM_POSTGRESQL_CHART) $(HELM_POSTGRESQL_VALUES_ARG) --namespace $(HELM_POSTGRESQL_NAMESPACE) --create-namespace \ + --set-file envFile.raw="$${COGSTACK_DATABASE_ENV_FILE}" \ + --set-file usersEnvFile.raw="$${COGSTACK_USERS_DATABASE_ENV_FILE}" helm-repo-add-gitea: ## Add/update the official Gitea Helm repository @if ! helm repo list | awk '$$1 == "$(HELM_GITEA_REPO_NAME)" { found=1 } END { exit !found }'; then \ @@ -301,15 +323,23 @@ _check-remote-params: .PHONY: remote-deploy-service remote-stop-service remote-delete-service _check-remote-params helm-template-opensearch helm-install-opensearch kube-install-cnpg-operator helm-template-nifi helm-install-nifi helm-template-postgresql helm-install-postgresql helm-repo-add-gitea helm-template-gitea helm-install-gitea +check-nifi-resources: ## Validate NiFi heap sizes against the container memory limit + @$(WITH_ENV) python3 ../scripts/tests/check_nifi_resources.py + +check-config: check-nifi-resources ## Validate resolved settings and both core Compose files + @$(WITH_ENV) docker compose -f services.yml config --quiet && docker compose -f services.dev.yml config --quiet + +.PHONY: check-nifi-resources check-config + # start services -start-nifi: ## Start NiFi and NiFi nginx (services.yml) +start-nifi: check-nifi-resources ## Start NiFi and NiFi nginx (services.yml) $(WITH_ENV) docker compose -f services.yml $(DC_START_CMD) nifi nifi-nginx -start-nifi-dev: ## Start NiFi and NiFi nginx (services.dev.yml) +start-nifi-dev: check-nifi-resources ## Start NiFi and NiFi nginx (services.dev.yml) $(WITH_ENV) docker compose -f services.dev.yml $(DC_START_CMD) nifi nifi-nginx -start-nifi-dev-build: ## Build and start NiFi dev services +start-nifi-dev-build: check-nifi-resources ## Build and start NiFi dev services $(WITH_ENV) docker compose -f services.dev.yml up -d --build nifi nifi-nginx build-nifi-image: ## Build the local NiFi image without starting services diff --git a/deploy/charts/nifi/templates/_helpers.tpl b/deploy/charts/nifi/templates/_helpers.tpl index a0e9dffd3..e16490377 100644 --- a/deploy/charts/nifi/templates/_helpers.tpl +++ b/deploy/charts/nifi/templates/_helpers.tpl @@ -155,6 +155,23 @@ app.kubernetes.io/component: nifi {{ toYaml $usersData }} {{- end -}} +{{/* Convert JVM and Kubernetes memory quantities to bytes for resource validation. */}} +{{- define "cogstack-nifi.memoryBytes" -}} +{{- $value := lower (trim (printf "%v" .)) -}} +{{- if not (regexMatch "^[0-9]+(\\.[0-9]+)?([kmgtpe](i?b?)?)?$" $value) -}} +{{- fail "NiFi memory sizes must use positive quantities such as 768m, 1g, or 2Gi" -}} +{{- end -}} +{{- $number := regexFind "^[0-9]+(\\.[0-9]+)?" $value -}} +{{- $suffix := trimPrefix $number $value -}} +{{- $unit := regexFind "^[kmgtpe]" $suffix -}} +{{- $multipliers := dict "" 1 "k" 1024 "m" 1048576 "g" 1073741824 "t" 1099511627776 "p" 1125899906842624 "e" 1152921504606846976 -}} +{{- $bytes := mulf (float64 $number) (float64 (index $multipliers $unit)) -}} +{{- if le $bytes (float64 0) -}} +{{- fail "NiFi memory sizes must be positive" -}} +{{- end -}} +{{- printf "%.0f" $bytes -}} +{{- end -}} + {{/* Service names */}} {{- define "cogstack-nifi.serviceName" -}} {{- printf "%s" (include "cogstack-nifi.fullname" .) -}} diff --git a/deploy/charts/nifi/templates/statefulset.yaml b/deploy/charts/nifi/templates/statefulset.yaml index d8612abf9..0f81ed03d 100644 --- a/deploy/charts/nifi/templates/statefulset.yaml +++ b/deploy/charts/nifi/templates/statefulset.yaml @@ -94,6 +94,20 @@ {{- if hasKey $envData "NIFI_DOCKER_RAM" -}} {{- $memoryLimit = include "cogstack-nifi.normalizeMemory" (index $envData "NIFI_DOCKER_RAM") -}} {{- end -}} +{{- range $heap := list $heapInit $heapMax -}} +{{- if not (regexMatch "^[0-9]+[kKmMgG]?$" (printf "%v" $heap)) -}} +{{- fail "NiFi JVM heap sizes must use positive integers with optional k, m, or g suffixes" -}} +{{- end -}} +{{- end -}} +{{- $heapInitBytes := include "cogstack-nifi.memoryBytes" $heapInit | float64 -}} +{{- $heapMaxBytes := include "cogstack-nifi.memoryBytes" $heapMax | float64 -}} +{{- $memoryLimitBytes := include "cogstack-nifi.memoryBytes" $memoryLimit | float64 -}} +{{- if gt $heapInitBytes $heapMaxBytes -}} +{{- fail "NIFI_JVM_HEAP_INIT must not exceed NIFI_JVM_HEAP_MAX" -}} +{{- end -}} +{{- if ge $heapMaxBytes $memoryLimitBytes -}} +{{- fail "NIFI_JVM_HEAP_MAX must be smaller than NIFI_DOCKER_RAM; leave memory for native allocations and Python processors" -}} +{{- end -}} apiVersion: apps/v1 kind: StatefulSet metadata: diff --git a/deploy/charts/nifi/values.yaml b/deploy/charts/nifi/values.yaml index eb09d9e65..058f2edce 100644 --- a/deploy/charts/nifi/values.yaml +++ b/deploy/charts/nifi/values.yaml @@ -79,10 +79,10 @@ initSecurityContext: {} resources: requests: cpu: "1" - memory: 1Gi + memory: 2Gi limits: cpu: "1" - memory: 1Gi + memory: 2Gi terminationGracePeriodSeconds: 120 diff --git a/deploy/config_files.sh b/deploy/config_files.sh new file mode 100755 index 000000000..8a716447c --- /dev/null +++ b/deploy/config_files.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# Shared file selection for deployment settings and security helpers. + +cogstack_config_init() { + COGSTACK_REPO_ROOT="$1" + if [[ -n "${COGSTACK_CONFIG_DIR:-}" ]]; then + case "$COGSTACK_CONFIG_DIR" in + /*) ;; + *) COGSTACK_CONFIG_DIR="$COGSTACK_REPO_ROOT/$COGSTACK_CONFIG_DIR" ;; + esac + if [[ ! -d "$COGSTACK_CONFIG_DIR" ]]; then + echo "ERROR: COGSTACK_CONFIG_DIR is not a directory: $COGSTACK_CONFIG_DIR" >&2 + return 1 + fi + COGSTACK_CONFIG_DIR="$(cd "$COGSTACK_CONFIG_DIR" && pwd)" || return 1 + fi + export COGSTACK_CONFIG_DIR +} + +cogstack_env_file() { + local relative_path="$1" + local selected_path="${2:-$COGSTACK_REPO_ROOT/$relative_path}" + local custom_path="" + + if [[ -n "${COGSTACK_CONFIG_DIR:-}" ]]; then + case "$relative_path" in + deploy/*) custom_path="$COGSTACK_CONFIG_DIR/${relative_path#deploy/}" ;; + security/env/*) custom_path="$COGSTACK_CONFIG_DIR/security/${relative_path#security/env/}" ;; + esac + if [[ -n "$custom_path" && ( -e "$custom_path" || -L "$custom_path" ) ]]; then + selected_path="$custom_path" + fi + fi + if [[ ! -f "$selected_path" || ! -r "$selected_path" ]]; then + echo "ERROR: Configuration file is not a readable regular file: $selected_path" >&2 + return 1 + fi + printf '%s\n' "$selected_path" +} diff --git a/deploy/export_env_vars.sh b/deploy/export_env_vars.sh index db59b8662..40c0a8aa0 100755 --- a/deploy/export_env_vars.sh +++ b/deploy/export_env_vars.sh @@ -1,67 +1,90 @@ #!/usr/bin/env bash +# Source selected configuration without changing the caller's shell options. -# Enable strict mode (without -e to avoid exit-on-error) -set -uo pipefail - -# Support being sourced in shells where BASH_SOURCE is unset (e.g. zsh) -SCRIPT_SOURCE="${BASH_SOURCE[0]-$0}" -SCRIPT_DIR="$(cd "$(dirname "$SCRIPT_SOURCE")" && pwd)" -SCRIPT_NAME="$(basename "$SCRIPT_SOURCE")" - -echo "🔧 Running $SCRIPT_NAME..." - -set -a - -DEPLOY_DIR="$SCRIPT_DIR" -SECURITY_DIR="$SCRIPT_DIR/../security/env" -SERVICES_DIR="$SCRIPT_DIR/../services" - - -env_files=( - "$SECURITY_DIR/certificates_nifi.env" - "$SECURITY_DIR/certificates_general.env" - "$SECURITY_DIR/certificates_elasticsearch.env" - "$SECURITY_DIR/users_elasticsearch.env" - "$SECURITY_DIR/users_database.env" - "$SECURITY_DIR/users_nifi.env" - "$SECURITY_DIR/users_nginx.env" - - "$DEPLOY_DIR/general.env" - "$DEPLOY_DIR/nifi.env" - "$DEPLOY_DIR/elasticsearch.env" - "$DEPLOY_DIR/ollama.env" - "$DEPLOY_DIR/sglang.env" - "$DEPLOY_DIR/openwebui.env" - "$DEPLOY_DIR/database.env" - "$DEPLOY_DIR/network_settings.env" - "$DEPLOY_DIR/project.env" - "$DEPLOY_DIR/gitea.env" - "$DEPLOY_DIR/nginx.env" - "$DEPLOY_DIR/telemetry.env" - - "$SERVICES_DIR/cogstack-platform/apps/cogstack-jupyter-hub/env/jupyter.env" - "$SERVICES_DIR/cogstack-platform/apps/ocr-service/env/ocr_service.env" - "$SERVICES_DIR/cogstack-nlp/medcat-service/env/app.env" - "$SERVICES_DIR/cogstack-nlp/medcat-service/env/medcat.env" - "$SERVICES_DIR/cogstack-nlp/medcat-trainer/envs/env-prod" -) - -for env_file in "${env_files[@]}"; do - if [ -f "$env_file" ]; then - echo "✅ Sourcing $env_file" - # shellcheck disable=SC1090 - source "$env_file" +_cogstack_load_environment() { + local loader_path + if [[ -n "${ZSH_VERSION:-}" ]]; then + # The expression is evaluated only by zsh; Bash uses BASH_SOURCE below. + eval 'loader_path=${(%):-%x}' else - echo "⚠️ Skipping missing env file: $env_file" + loader_path="${BASH_SOURCE[0]}" fi -done + local deploy_path + deploy_path="$(cd "$(dirname "$loader_path")" && pwd)" || return 1 + # shellcheck disable=SC1091 + source "$deploy_path/config_files.sh" || return 1 + cogstack_config_init "$(cd "$deploy_path/.." && pwd)" || return 1 -# for nginx vars -export DOLLAR="$" + local relative_files=( + security/env/certificates_nifi.env + security/env/certificates_general.env + security/env/certificates_elasticsearch.env + security/env/users_elasticsearch.env + security/env/users_database.env + security/env/users_nifi.env + security/env/users_nginx.env + deploy/general.env + deploy/nifi.env + deploy/elasticsearch.env + deploy/ollama.env + deploy/sglang.env + deploy/openwebui.env + deploy/database.env + deploy/network_settings.env + deploy/project.env + deploy/gitea.env + deploy/nginx.env + deploy/telemetry.env + ) + local relative_file selected_file variable_name file_name + local restore_allexport=1 + case "$-" in *a*) restore_allexport=0 ;; esac + set -a + for relative_file in "${relative_files[@]}"; do + selected_file="$(cogstack_env_file "$relative_file")" || { + if [[ "$restore_allexport" == 1 ]]; then set +a; fi + return 1 + } + file_name="${relative_file##*/}" + variable_name="COGSTACK_$(printf '%s' "${file_name%.env}" | tr '[:lower:]' '[:upper:]')_ENV_FILE" + export "$variable_name=$selected_file" + # shellcheck disable=SC1090 + source "$selected_file" || { + echo "ERROR: Failed to load configuration: $selected_file" >&2 + if [[ "$restore_allexport" == 1 ]]; then set +a; fi + return 1 + } + done -# Disable auto-export -set +a + # Service-owned configuration remains optional and uses its own directories. + local service_files=( + services/cogstack-platform/apps/cogstack-jupyter-hub/env/jupyter.env + services/cogstack-platform/apps/ocr-service/env/ocr_service.env + services/cogstack-nlp/medcat-service/env/app.env + services/cogstack-nlp/medcat-service/env/medcat.env + services/cogstack-nlp/medcat-trainer/envs/env-prod + ) + for relative_file in "${service_files[@]}"; do + selected_file="$COGSTACK_REPO_ROOT/$relative_file" + if [[ -f "$selected_file" ]]; then + # shellcheck disable=SC1090 + source "$selected_file" || { + echo "ERROR: Failed to load service configuration: $selected_file" >&2 + if [[ "$restore_allexport" == 1 ]]; then set +a; fi + return 1 + } + fi + done + export DOLLAR='$' + if [[ "$restore_allexport" == 1 ]]; then set +a; fi + echo "Configuration loaded from ${COGSTACK_CONFIG_DIR:-repository defaults}." +} -# Restore safe defaults for interactive/dev shell -set +u -set +o pipefail +if _cogstack_load_environment; then + unset -f _cogstack_load_environment +else + unset -f _cogstack_load_environment + # Also support executing the loader instead of sourcing it. + # shellcheck disable=SC2317 + return 1 2>/dev/null || exit 1 +fi diff --git a/deploy/general.env b/deploy/general.env index bd27cd8ff..843333f2e 100644 --- a/deploy/general.env +++ b/deploy/general.env @@ -4,7 +4,7 @@ # set only CPU_ARCHITECTURE to amd/arm64 # remove if this causes issues on any other platform # possible values: amd64, arm64 -CPU_ARCHITECTURE=arm64 +CPU_ARCHITECTURE=amd64 DOCKER_DEFAULT_PLATFORM=linux/${CPU_ARCHITECTURE:-amd64} COMPOSE_BAKE=True diff --git a/deploy/nifi.env b/deploy/nifi.env index ca229cee7..c06995f8c 100644 --- a/deploy/nifi.env +++ b/deploy/nifi.env @@ -5,7 +5,7 @@ ############################################################################################################################## NIFI_JVM_OPTS="-XX:+UseG1GC -XX:MaxGCPauseMillis=250 -XX:+ParallelRefProcEnabled -Xlog:gc*:file=/opt/nifi/nifi-current/logs/gc.log:time,uptime,level,tags:filecount=5,filesize=50m -Djava.security.egd=file:/dev/./urandom" NIFI_JVM_HEAP_INIT=768m -NIFI_JVM_HEAP_MAX=2g +NIFI_JVM_HEAP_MAX=1g NIFI_DOCKER_SHM_SIZE=512m diff --git a/deploy/services.dev.yml b/deploy/services.dev.yml index 94cf738ce..f9a507ce9 100644 --- a/deploy/services.dev.yml +++ b/deploy/services.dev.yml @@ -9,32 +9,32 @@ x-nifi-logging-common: &nifi-logging-common max-file: ${NIFI_DOCKER_LOG_NUM_FILES:-10} x-all-env: &all-env - - ./project.env - - ./general.env - - ./nifi.env - - ./gitea.env - - ./nginx.env - - ./telemetry.env - - ./database.env - - ./elasticsearch.env - - ./network_settings.env - - ../security/env/users_nifi.env - - ../security/env/users_database.env - - ../security/env/users_nginx.env - - ../security/env/users_elasticsearch.env - - ../security/env/certificates_general.env - - ../security/env/certificates_elasticsearch.env - - ../security/env/certificates_nifi.env + - "${COGSTACK_PROJECT_ENV_FILE:-./project.env}" + - "${COGSTACK_GENERAL_ENV_FILE:-./general.env}" + - "${COGSTACK_NIFI_ENV_FILE:-./nifi.env}" + - "${COGSTACK_GITEA_ENV_FILE:-./gitea.env}" + - "${COGSTACK_NGINX_ENV_FILE:-./nginx.env}" + - "${COGSTACK_TELEMETRY_ENV_FILE:-./telemetry.env}" + - "${COGSTACK_DATABASE_ENV_FILE:-./database.env}" + - "${COGSTACK_ELASTICSEARCH_ENV_FILE:-./elasticsearch.env}" + - "${COGSTACK_NETWORK_SETTINGS_ENV_FILE:-./network_settings.env}" + - "${COGSTACK_USERS_NIFI_ENV_FILE:-../security/env/users_nifi.env}" + - "${COGSTACK_USERS_DATABASE_ENV_FILE:-../security/env/users_database.env}" + - "${COGSTACK_USERS_NGINX_ENV_FILE:-../security/env/users_nginx.env}" + - "${COGSTACK_USERS_ELASTICSEARCH_ENV_FILE:-../security/env/users_elasticsearch.env}" + - "${COGSTACK_CERTIFICATES_GENERAL_ENV_FILE:-../security/env/certificates_general.env}" + - "${COGSTACK_CERTIFICATES_ELASTICSEARCH_ENV_FILE:-../security/env/certificates_elasticsearch.env}" + - "${COGSTACK_CERTIFICATES_NIFI_ENV_FILE:-../security/env/certificates_nifi.env}" x-es-env: &es-env - - ./network_settings.env - - ./elasticsearch.env - - ../security/env/users_elasticsearch.env - - ../security/env/certificates_elasticsearch.env + - "${COGSTACK_NETWORK_SETTINGS_ENV_FILE:-./network_settings.env}" + - "${COGSTACK_ELASTICSEARCH_ENV_FILE:-./elasticsearch.env}" + - "${COGSTACK_USERS_ELASTICSEARCH_ENV_FILE:-../security/env/users_elasticsearch.env}" + - "${COGSTACK_CERTIFICATES_ELASTICSEARCH_ENV_FILE:-../security/env/certificates_elasticsearch.env}" x-db-env: &db-env - - ./database.env - - ../security/env/users_database.env + - "${COGSTACK_DATABASE_ENV_FILE:-./database.env}" + - "${COGSTACK_USERS_DATABASE_ENV_FILE:-../security/env/users_database.env}" x-common-hosts: &common-hosts - ${ELASTICSEARCH_1_HOST_NAME:-test-1:0.0.0.0} @@ -82,10 +82,10 @@ x-nifi-volumes: &nifi-volumes - ../nifi/conf/:/opt/nifi/nifi-current/conf/:rw # Ingest data directory - - ./${NIFI_DATA_PATH:-../data/}:/data/:rw + - ${NIFI_DATA_PATH:-../data/}:/data/:rw # MedCAT models - - ./${NIFI_MEDCAT_SERVICE_MODEL_PRODUCTION_PATH:-../services/cogstack-nlp/medcat-service/models/}:/opt/models:rw + - ${NIFI_MEDCAT_SERVICE_MODEL_PRODUCTION_PATH:-../services/cogstack-nlp/medcat-service/models/}:/opt/models:rw # NiFi repositories/state - nifi-vol-logs:/opt/nifi/nifi-current/logs @@ -121,7 +121,7 @@ services: shm_size: ${NIFI_DOCKER_SHM_SIZE:-"1g"} environment: - PYTHONPATH=${NIFI_PYTHONPATH:-/opt/nifi/nifi-current/python/framework} - - JVM_OPTS="${NIFI_JVM_OPTS:--XX:+UseG1GC -XX:MaxGCPauseMillis=200 -XX:+ParallelRefProcEnabled -Djava.security.egd=file:/dev/./urandom}" + - JVM_OPTS=${NIFI_JVM_OPTS:--XX:+UseG1GC -XX:MaxGCPauseMillis=200 -XX:+ParallelRefProcEnabled -Djava.security.egd=file:/dev/./urandom} deploy: resources: limits: diff --git a/deploy/services.yml b/deploy/services.yml index 9f5ae736a..c68a801da 100644 --- a/deploy/services.yml +++ b/deploy/services.yml @@ -20,32 +20,32 @@ x-logging-common: &logging-common max-file: ${DOCKER_LOG_NUM_FILES:-10} x-all-env: &all-env - - ./project.env - - ./general.env - - ./nifi.env - - ./gitea.env - - ./nginx.env - - ./telemetry.env - - ./database.env - - ./elasticsearch.env - - ./network_settings.env - - ../security/env/users_nifi.env - - ../security/env/users_database.env - - ../security/env/users_nginx.env - - ../security/env/users_elasticsearch.env - - ../security/env/certificates_general.env - - ../security/env/certificates_elasticsearch.env - - ../security/env/certificates_nifi.env + - "${COGSTACK_PROJECT_ENV_FILE:-./project.env}" + - "${COGSTACK_GENERAL_ENV_FILE:-./general.env}" + - "${COGSTACK_NIFI_ENV_FILE:-./nifi.env}" + - "${COGSTACK_GITEA_ENV_FILE:-./gitea.env}" + - "${COGSTACK_NGINX_ENV_FILE:-./nginx.env}" + - "${COGSTACK_TELEMETRY_ENV_FILE:-./telemetry.env}" + - "${COGSTACK_DATABASE_ENV_FILE:-./database.env}" + - "${COGSTACK_ELASTICSEARCH_ENV_FILE:-./elasticsearch.env}" + - "${COGSTACK_NETWORK_SETTINGS_ENV_FILE:-./network_settings.env}" + - "${COGSTACK_USERS_NIFI_ENV_FILE:-../security/env/users_nifi.env}" + - "${COGSTACK_USERS_DATABASE_ENV_FILE:-../security/env/users_database.env}" + - "${COGSTACK_USERS_NGINX_ENV_FILE:-../security/env/users_nginx.env}" + - "${COGSTACK_USERS_ELASTICSEARCH_ENV_FILE:-../security/env/users_elasticsearch.env}" + - "${COGSTACK_CERTIFICATES_GENERAL_ENV_FILE:-../security/env/certificates_general.env}" + - "${COGSTACK_CERTIFICATES_ELASTICSEARCH_ENV_FILE:-../security/env/certificates_elasticsearch.env}" + - "${COGSTACK_CERTIFICATES_NIFI_ENV_FILE:-../security/env/certificates_nifi.env}" x-es-env: &es-env - - ./network_settings.env - - ./elasticsearch.env - - ../security/env/users_elasticsearch.env - - ../security/env/certificates_elasticsearch.env + - "${COGSTACK_NETWORK_SETTINGS_ENV_FILE:-./network_settings.env}" + - "${COGSTACK_ELASTICSEARCH_ENV_FILE:-./elasticsearch.env}" + - "${COGSTACK_USERS_ELASTICSEARCH_ENV_FILE:-../security/env/users_elasticsearch.env}" + - "${COGSTACK_CERTIFICATES_ELASTICSEARCH_ENV_FILE:-../security/env/certificates_elasticsearch.env}" x-db-env: &db-env - - ./database.env - - ../security/env/users_database.env + - "${COGSTACK_DATABASE_ENV_FILE:-./database.env}" + - "${COGSTACK_USERS_DATABASE_ENV_FILE:-../security/env/users_database.env}" x-common-hosts: &common-hosts - ${ELASTICSEARCH_1_HOST_NAME:-test-1:0.0.0.0} @@ -96,10 +96,10 @@ x-nifi-volumes: &nifi-volumes - ../nifi/conf/:/opt/nifi/nifi-current/conf/:rw # Ingest data directory - - ./${NIFI_DATA_PATH:-../data/}:/data/:rw + - ${NIFI_DATA_PATH:-../data/}:/data/:rw # MedCAT models - - ./${NIFI_MEDCAT_SERVICE_MODEL_PRODUCTION_PATH:-../services/cogstack-nlp/medcat-service/models/}:/opt/models:rw + - ${NIFI_MEDCAT_SERVICE_MODEL_PRODUCTION_PATH:-../services/cogstack-nlp/medcat-service/models/}:/opt/models:rw # NiFi repositories/state - nifi-vol-logs:/opt/nifi/nifi-current/logs @@ -185,8 +185,8 @@ x-metricbeat-common: &metricbeat-common restart: unless-stopped shm_size: ${METRICBEAT_DOCKER_SHM:-1g} env_file: - - ./elasticsearch.env - - ../security/env/users_elasticsearch.env + - "${COGSTACK_ELASTICSEARCH_ENV_FILE:-./elasticsearch.env}" + - "${COGSTACK_USERS_ELASTICSEARCH_ENV_FILE:-../security/env/users_elasticsearch.env}" deploy: resources: limits: @@ -210,8 +210,8 @@ x-filebeat-common: &filebeat-common restart: unless-stopped shm_size: ${FILEBEAT_DOCKER_SHM:-1g} env_file: - - ./elasticsearch.env - - ../security/env/users_elasticsearch.env + - "${COGSTACK_ELASTICSEARCH_ENV_FILE:-./elasticsearch.env}" + - "${COGSTACK_USERS_ELASTICSEARCH_ENV_FILE:-../security/env/users_elasticsearch.env}" deploy: resources: limits: @@ -313,7 +313,7 @@ services: hostname: clickhouse restart: unless-stopped env_file: - - ./database.env + - "${COGSTACK_DATABASE_ENV_FILE:-./database.env}" ulimits: nofile: soft: 262144 @@ -556,7 +556,7 @@ services: platform: ${DOCKER_DEFAULT_PLATFORM:-linux/amd64} restart: unless-stopped env_file: - - ./ollama.env + - "${COGSTACK_OLLAMA_ENV_FILE:-./ollama.env}" deploy: resources: limits: @@ -625,7 +625,7 @@ services: platform: ${DOCKER_DEFAULT_PLATFORM:-linux/amd64} restart: unless-stopped env_file: - - ./openwebui.env + - "${COGSTACK_OPENWEBUI_ENV_FILE:-./openwebui.env}" depends_on: ollama: condition: service_healthy @@ -659,7 +659,7 @@ services: shm_size: ${NIFI_DOCKER_SHM_SIZE:-"1g"} environment: - PYTHONPATH=${NIFI_PYTHONPATH:-/opt/nifi/nifi-current/python/framework} - - JVM_OPTS="${NIFI_JVM_OPTS:--XX:+UseG1GC -XX:MaxGCPauseMillis=200 -XX:+ParallelRefProcEnabled -Djava.security.egd=file:/dev/./urandom}" + - JVM_OPTS=${NIFI_JVM_OPTS:--XX:+UseG1GC -XX:MaxGCPauseMillis=200 -XX:+ParallelRefProcEnabled -Djava.security.egd=file:/dev/./urandom} deploy: resources: limits: diff --git a/docs/deploy/configuration.md b/docs/deploy/configuration.md index 81ec39536..a1d5076d5 100644 --- a/docs/deploy/configuration.md +++ b/docs/deploy/configuration.md @@ -1,7 +1,7 @@ # ⚙️ Configuration -CogStack separates deployment configuration from local-development security -settings: +CogStack selects deployment configuration and security settings from the +repository defaults or a directory selected with `COGSTACK_CONFIG_DIR`: - `deploy/*.env` configures services, images, ports, resource limits, and the shared Docker network. @@ -25,6 +25,8 @@ The core Compose file, `deploy/services.yml`, loads these files: | `deploy/nifi.env` | NiFi image, ports, resources, proxy settings, and paths | | `deploy/elasticsearch.env` | Elasticsearch/OpenSearch, Dashboards/Kibana, and Beats | | `deploy/ollama.env` | Ollama image, model selection, port, and resource limits | +| `deploy/sglang.env` | Existing SGLang endpoint, served model, and API key | +| `deploy/openwebui.env` | Open WebUI image, access, and resource limits | | `deploy/database.env` | Sample and production database resources | | `deploy/gitea.env` | Gitea image, ports, and application settings | | `deploy/nginx.env` | NiFi nginx proxy settings | @@ -81,17 +83,110 @@ credentials and certificate passwords. ## Deployment-specific overrides -For a separate deployment, copy the tracked defaults to a protected location -and edit the copies there: +Copy the files you need to customize into a protected directory. Run these +commands from the repository root: ```bash -cp deploy/*.env /path/to/deployment-config/ -cp security/env/*.env /path/to/deployment-config/security/ +mkdir -p /path/to/deployment-config/security +cp deploy/nifi.env /path/to/deployment-config/nifi.env +cp security/env/users_nifi.env /path/to/deployment-config/security/users_nifi.env +cp security/env/certificates_nifi.env /path/to/deployment-config/security/certificates_nifi.env + +# Edit the copied files, then validate and start the deployment. +COGSTACK_CONFIG_DIR=/path/to/deployment-config make -C deploy check-config +COGSTACK_CONFIG_DIR=/path/to/deployment-config make -C deploy init-security +COGSTACK_CONFIG_DIR=/path/to/deployment-config make -C deploy start-data-infra ``` -You must then adapt your Compose invocation or deployment tooling to load those -copies. The repository Makefile always loads the paths listed in -`deploy/export_env_vars.sh`. +You can also copy all `deploy/*.env` files and all `security/env/*.env` files +into these two directories. Protect files containing credentials with the +permissions required by your deployment. + +For local configuration, `deploy/local/` is ignored by Git: + +```bash +mkdir -p deploy/local +cp deploy/nifi.env deploy/local/nifi.env +COGSTACK_CONFIG_DIR=deploy/local make -C deploy check-config +``` + +Relative `COGSTACK_CONFIG_DIR` paths are resolved from the repository root, +including when using `make -C deploy`. An explicit missing directory, an +unreadable selected file, or a broken profile symlink causes an error. + +### File selection and precedence + +| Setting | Selected file, in priority order | +|---|---| +| Core settings such as `nifi.env` | `$COGSTACK_CONFIG_DIR/nifi.env`, then `deploy/nifi.env` | +| Security settings such as `users_nifi.env` | `$COGSTACK_CONFIG_DIR/security/users_nifi.env`, then `security/env/users_nifi.env` | + +A matching profile file replaces the complete default file. Copy the original +before editing it; individual keys are not merged with the original. Files +absent from the profile use their repository defaults. Make, Compose, +certificate helpers, and the owned Helm Make targets use the selected files. + +Files are sourced as shell assignments. Plain `KEY=value` assignments replace +inherited values. Assignments such as `KEY=${KEY:-default}` honor an existing +value, as used by the SGLang settings. Use plain assignments for authoritative +profile values, and keep these trusted files compatible with Compose's +environment-file syntax. + +For direct Compose commands, select the profile before loading configuration: + +```bash +export COGSTACK_CONFIG_DIR=/path/to/deployment-config +source deploy/export_env_vars.sh +docker compose -f deploy/services.yml config --quiet +docker compose -f deploy/services.yml up -d nifi nifi-nginx +``` + +After editing a selected file, source the loader again before invoking Compose. +It exports `COGSTACK_*_ENV_FILE` paths for Compose and preserves the caller's +shell options. Explicit Compose `environment` entries and `docker compose run -e` +retain their normal [precedence](https://docs.docker.com/compose/how-tos/environment-variables/envvars-precedence/). + +Relative data and model mount paths remain relative to `deploy/`; absolute paths +are supported through `NIFI_DATA_PATH` and +`NIFI_MEDCAT_SERVICE_MODEL_PRODUCTION_PATH`. Certificate material is generated +and mounted under `security/certificates/` using the selected security settings. + +### Helm and standalone services + +The owned Helm Make targets pass selected files through `--set-file`: + +```bash +COGSTACK_CONFIG_DIR=/path/to/deployment-config make -C deploy helm-template-nifi +``` + +Use `HELM_NIFI_VALUES_FILE`, `HELM_OPENSEARCH_VALUES_FILE`, or +`HELM_POSTGRESQL_VALUES_FILE` for cluster-specific values. Direct Helm commands +should supply selected files with `--set-file envFile.raw`, +`--set-file usersEnvFile.raw`, and, where used, `--set-file certificatesEnvFile.raw`. + +Standalone services in submodules retain their own environment configuration. + +## NiFi memory defaults and validation + +| Setting | Default | +|---|---| +| Initial JVM heap | `NIFI_JVM_HEAP_INIT=768m` | +| Maximum JVM heap | `NIFI_JVM_HEAP_MAX=1g` | +| Container memory limit | `NIFI_DOCKER_RAM=2g` | + +The remaining memory accommodates native JVM allocations and Python processors. +Tune these values together for the workload; see the +[NiFi sizing guidance](../nifi/main.md#critical-deploynifienv-settings). + +`make -C deploy check-config` validates the resolved memory settings and both +core Compose files without starting services. NiFi start targets run the memory +check automatically. Direct Helm rendering also rejects inconsistent heap and +limit values. + +Validation requires positive memory sizes, an initial heap no larger than the +maximum heap, and a maximum heap smaller than the container limit. JVM heaps +use whole numbers with optional `k`, `m`, or `g` suffixes. Container memory +also accepts units such as `GiB`. ## Multiple deployments on one host diff --git a/docs/deploy/deployment.md b/docs/deploy/deployment.md index 015df7809..940ef3868 100644 --- a/docs/deploy/deployment.md +++ b/docs/deploy/deployment.md @@ -252,6 +252,8 @@ make -C deploy remote-delete-service \ | Command | Description | |------------------------|---------------------------------------------| | `make load-env` | Load all environment variables | +| `make check-config` | Validate selected configuration, NiFi memory, and both core Compose files | +| `make check-nifi-resources` | Check JVM heap sizes against the container memory limit | | `make show-env` | Print environment variables (sorted) | | `make init-security` | Generate missing NiFi, Gitea, and selected search-backend certificates | | `make init-security-root-ca` | Generate the shared root CA if missing | diff --git a/docs/deploy/reproducible.md b/docs/deploy/reproducible.md index e6657cc91..5cff7d085 100644 --- a/docs/deploy/reproducible.md +++ b/docs/deploy/reproducible.md @@ -13,6 +13,7 @@ Run these commands from the repository root: ```bash git lfs pull make -C deploy git-init-submodules +make -C deploy check-config make -C deploy init-security make -C deploy start-data-infra ``` diff --git a/docs/index.md b/docs/index.md index 9f43a6ff3..9256fc706 100644 --- a/docs/index.md +++ b/docs/index.md @@ -62,6 +62,7 @@ Navigate to the root folder of the repository git lfs pull make -C deploy git-init-submodules make -C deploy help +make -C deploy check-config make -C deploy init-security make -C deploy start-data-infra ``` diff --git a/docs/nifi/main.md b/docs/nifi/main.md index 248e18cfd..52cd70999 100644 --- a/docs/nifi/main.md +++ b/docs/nifi/main.md @@ -84,7 +84,15 @@ NIFI_DOCKER_RAM=6g NIFI_DOCKER_SHM_SIZE=2g ``` -Note: the current defaults in `deploy/nifi.env` are small development-oriented values, so production deployments should usually increase these settings. +The reference profile uses a `768m` initial heap, a `1g` maximum heap, and a `2g` +container limit. Larger document batches or additional Python processors may +need more memory. Put deployment-specific settings in a copied `nifi.env` +selected with `COGSTACK_CONFIG_DIR`; see +[configuration profiles](../deploy/configuration.md#deployment-specific-overrides). + +`make -C deploy check-config` validates these settings before startup. NiFi start +targets also reject a heap that reaches the container limit, or an initial +heap larger than the maximum heap. For much more detailed information please refer to the official [Apache NiFi System Administrator's Guide](https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html). Another good reference for these properties is the [Cloudera Doc](https://docs.cloudera.com/HDPDocuments/HDF3/HDF-3.3.1/nifi-system-properties/hdf-nifi-system-properties.pdf) which describes everything in detail.
diff --git a/scripts/opensearch-ai-init.sh b/scripts/opensearch-ai-init.sh index 32d383490..0e2c86610 100755 --- a/scripts/opensearch-ai-init.sh +++ b/scripts/opensearch-ai-init.sh @@ -5,6 +5,10 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +# shellcheck disable=SC1091 +source "$PROJECT_DIR/deploy/config_files.sh" +cogstack_config_init "$PROJECT_DIR" + log() { printf '[opensearch-ai-init] %s\n' "$*" >&2 } @@ -15,7 +19,8 @@ die() { } load_env_file() { - local env_file="$1" + local env_file + env_file="$(cogstack_env_file "${1#"$PROJECT_DIR/"}")" || return 1 if [[ -f "$env_file" ]]; then set -a # shellcheck disable=SC1090 diff --git a/scripts/tests/check_nifi_resources.py b/scripts/tests/check_nifi_resources.py new file mode 100755 index 000000000..100cefafc --- /dev/null +++ b/scripts/tests/check_nifi_resources.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +"""Validate resolved NiFi heap and container memory settings before startup.""" + +from __future__ import annotations + +import os +import re +import sys +from collections.abc import Mapping +from decimal import Decimal + + +def memory_bytes(value: str, *, heap: bool = False) -> Decimal: + pattern = r"([0-9]+)([kmg]?)" if heap else r"([0-9]+(?:\.[0-9]+)?)([kmgtpe]?)(?:i?b)?" + match = re.fullmatch(pattern, value.strip(), re.IGNORECASE) + if not match: + raise ValueError("use a positive memory size such as 768m, 1g, or 2GiB") + size = Decimal(match[1]) * 1024 ** ("kmgtpe".index(match[2].lower()) + 1 if match[2] else 0) + if size <= 0: + raise ValueError("memory sizes must be positive") + return size + + +def resource_errors(values: Mapping[str, str]) -> list[str]: + sizes: dict[str, Decimal] = {} + errors = [] + for key in ("NIFI_JVM_HEAP_INIT", "NIFI_JVM_HEAP_MAX", "NIFI_DOCKER_RAM"): + try: + sizes[key] = memory_bytes(values.get(key, ""), heap=key != "NIFI_DOCKER_RAM") + except ValueError as exc: + errors.append(f"{key}: {exc}") + if errors: + return errors + if sizes["NIFI_JVM_HEAP_INIT"] > sizes["NIFI_JVM_HEAP_MAX"]: + errors.append("NIFI_JVM_HEAP_INIT must not exceed NIFI_JVM_HEAP_MAX") + if sizes["NIFI_JVM_HEAP_MAX"] >= sizes["NIFI_DOCKER_RAM"]: + errors.append( + "NIFI_JVM_HEAP_MAX must be smaller than NIFI_DOCKER_RAM; " + "leave memory for native allocations and Python processors" + ) + return errors + + +def main() -> int: + errors = resource_errors(os.environ) + if errors: + print("Invalid NiFi resource configuration:", file=sys.stderr) + for error in errors: + print(f"- {error}", file=sys.stderr) + return 1 + print( + f"NiFi resources valid: heap {os.environ['NIFI_JVM_HEAP_INIT']}–{os.environ['NIFI_JVM_HEAP_MAX']}, " + f"container limit {os.environ['NIFI_DOCKER_RAM']}." + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/tests/test_configuration.py b/scripts/tests/test_configuration.py new file mode 100755 index 000000000..3a0ffa42e --- /dev/null +++ b/scripts/tests/test_configuration.py @@ -0,0 +1,188 @@ +#!/usr/bin/env python3 +"""Check profile selection, container values, and startup resource guards.""" + +from __future__ import annotations + +import json +import os +import re +import shutil +import subprocess +import tempfile +import unittest +from pathlib import Path + +from check_nifi_resources import resource_errors + +ROOT = Path(__file__).resolve().parents[2] + + +class ConfigurationTest(unittest.TestCase): + def setUp(self) -> None: + self.temp = tempfile.TemporaryDirectory(prefix="cogstack configuration ") + self.addCleanup(self.temp.cleanup) + self.directory = Path(self.temp.name) + self.profile = self.directory / "profile with spaces" + self.profile.mkdir() + self.env = {key: value for key, value in os.environ.items() if not key.startswith("COGSTACK_")} + self.env["COGSTACK_CONFIG_DIR"] = str(self.profile) + + def settings(self, name: str, changes: dict[str, str]) -> Path: + relative = f"security/env/{name}" if name.startswith(("users_", "certificates_")) else f"deploy/{name}" + target = self.profile / ("security" if relative.startswith("security/") else "") / name + target.parent.mkdir(parents=True, exist_ok=True) + text = (ROOT / relative).read_text() + for key, value in changes.items(): + replacement = f"{key}='{value}'" + text, count = re.subn( + rf"^{key}=.*$", replacement.replace("\\", "\\\\"), text, flags=re.MULTILINE, + ) + self.assertEqual(count, 1, key) + target.write_text(text) + return target + + def run_bash(self, command: str, *args: str, shell: str = "bash") -> subprocess.CompletedProcess[str]: + return subprocess.run( + [shell, "-c", command, "test-configuration", str(ROOT / "deploy/export_env_vars.sh"), *args], + env=self.env, cwd=self.directory, text=True, capture_output=True, timeout=30, + ) + + def test_selected_files_replace_defaults_and_missing_files_fall_back(self) -> None: + custom = self.settings("nifi.env", {"NIFI_JVM_HEAP_MAX": "1536m", "NIFI_DOCKER_RAM": "3g"}) + result = self.run_bash( + 'set -euo pipefail; source "$1" >/dev/null; ' + 'printf "%s\\n" "$NIFI_JVM_HEAP_MAX" "$NIFI_DOCKER_RAM" ' + '"$COGSTACK_NIFI_ENV_FILE" "$COGSTACK_DATABASE_ENV_FILE"' + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout.splitlines(), ["1536m", "3g", str(custom), str(ROOT / "deploy/database.env")]) + + @unittest.skipUnless(shutil.which("docker"), "Docker Compose is required") + def test_compose_uses_profile_for_resources_ports_and_container_environment(self) -> None: + self.settings("nifi.env", { + "NIFI_JVM_HEAP_MAX": "1536m", "NIFI_DOCKER_RAM": "3g", "NIFI_OUTPUT_PORT": "18082", + "NIFI_DATA_PATH": str(self.directory / "data with spaces"), + }) + selected_password = "fixture-$value with spaces" + self.settings("users_nifi.env", {"NIFI_SINGLE_USER_CREDENTIALS_PASSWORD": selected_password}) + for compose_file in ("services.yml", "services.dev.yml"): + with self.subTest(compose_file=compose_file): + result = self.run_bash( + 'source "$1" >/dev/null && docker compose -f "$2" config --format json', + str(ROOT / "deploy" / compose_file), + ) + self.assertEqual(result.returncode, 0, result.stderr) + nifi = json.loads(result.stdout)["services"]["nifi"] + self.assertEqual(nifi["environment"]["NIFI_JVM_HEAP_MAX"], "1536m") + # Compose escapes dollar signs when serializing a reusable model. + password = nifi["environment"]["NIFI_SINGLE_USER_CREDENTIALS_PASSWORD"].replace("$$", "$") + self.assertTrue(password == selected_password) + self.assertEqual(int(nifi["deploy"]["resources"]["limits"]["memory"]), 3 * 1024**3) + self.assertTrue(any(port.get("published") == "18082" for port in nifi["ports"])) + data_mount = next(volume for volume in nifi["volumes"] if volume["target"] == "/data") + self.assertEqual(data_mount["source"], str(self.directory / "data with spaces")) + self.assertFalse(nifi["environment"]["JVM_OPTS"].startswith('"')) + + def test_relative_profile_path_is_resolved_from_repository_root(self) -> None: + self.settings("nifi.env", {"NIFI_JVM_HEAP_MAX": "1536m", "NIFI_DOCKER_RAM": "3g"}) + self.env["COGSTACK_CONFIG_DIR"] = os.path.relpath(self.profile, ROOT) + result = self.run_bash('source "$1" >/dev/null; printf "%s\\n" "$COGSTACK_CONFIG_DIR"') + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(Path(result.stdout.strip()).resolve(), self.profile.resolve()) + result = self.make("check-nifi-resources") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("1536m", result.stdout) + + def test_loader_preserves_shell_options(self) -> None: + for flags in ("set -eu; set -o pipefail", "set -eua; set -o pipefail"): + result = self.run_bash( + flags + '; before="$-"; source "$1" >/dev/null; ' + 'test "$before" = "$-"; test -o pipefail; test -o nounset; test -o errexit' + ) + self.assertEqual(result.returncode, 0, result.stderr) + + @unittest.skipUnless(shutil.which("zsh"), "zsh is optional") + def test_loader_can_be_sourced_from_zsh(self) -> None: + result = self.run_bash('source "$1" >/dev/null && test "$NIFI_JVM_HEAP_MAX" = 1g', shell="zsh") + self.assertEqual(result.returncode, 0, result.stderr) + + def fake_docker(self) -> Path: + binary = self.directory / "docker" + log = self.directory / "docker-called" + binary.write_text('#!/bin/sh\n: > "$CONFIG_TEST_DOCKER_LOG"\nexit 99\n') + binary.chmod(0o755) + self.env["CONFIG_TEST_DOCKER_LOG"] = str(log) + self.env["PATH"] = f"{self.directory}:{self.env['PATH']}" + return log + + def make(self, target: str, *args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["make", "-s", "-C", str(ROOT / "deploy"), target, *args], + env=self.env, text=True, capture_output=True, timeout=30, + ) + + def test_invalid_directory_stops_make_before_docker(self) -> None: + log = self.fake_docker() + self.env["COGSTACK_CONFIG_DIR"] = str(self.directory / "missing") + result = self.make("check-config") + self.assertNotEqual(result.returncode, 0) + self.assertIn("not a directory", result.stderr) + self.assertFalse(log.exists()) + + def test_broken_profile_symlink_fails_instead_of_using_defaults(self) -> None: + (self.profile / "nifi.env").symlink_to(self.directory / "missing.env") + result = self.run_bash('source "$1"') + self.assertNotEqual(result.returncode, 0) + self.assertIn("not a readable regular file", result.stderr) + + def test_invalid_memory_stops_startup_before_docker(self) -> None: + log = self.fake_docker() + self.settings("nifi.env", {"NIFI_JVM_HEAP_MAX": "2g", "NIFI_DOCKER_RAM": "2g"}) + result = self.make("start-nifi") + self.assertNotEqual(result.returncode, 0) + self.assertIn("must be smaller", result.stderr) + self.assertFalse(log.exists()) + + def test_certificate_checks_use_selected_security_settings(self) -> None: + self.settings("certificates_general.env", {"ROOT_CERTIFICATE_NAME": "profile-root"}) + certificates = self.directory / "certificates" + root = certificates / "root" + root.mkdir(parents=True) + for suffix in (".key", ".pem", ".p12", "-keystore.jks", "-truststore.jks"): + (root / f"profile-root{suffix}").touch() + result = self.make("init-security-root-ca", f"SECURITY_CERTIFICATES_DIR={certificates}") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("already exist", result.stdout) + + @unittest.skipUnless(shutil.which("helm"), "Helm is required") + def test_helm_make_target_uses_selected_nifi_settings(self) -> None: + self.settings("nifi.env", {"NIFI_JVM_HEAP_MAX": "1536m", "NIFI_DOCKER_RAM": "3g"}) + result = self.make("helm-template-nifi") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn('"-Xmx1536m"', result.stdout) + self.assertIn('memory: "3Gi"', result.stdout) + + @unittest.skipUnless(shutil.which("helm"), "Helm is required") + def test_direct_helm_rejects_heap_equal_to_memory_limit(self) -> None: + settings = self.directory / "invalid.env" + settings.write_text("NIFI_JVM_HEAP_INIT=1g\nNIFI_JVM_HEAP_MAX=2g\nNIFI_DOCKER_RAM=2g\n") + result = subprocess.run( + ["helm", "template", "invalid", str(ROOT / "deploy/charts/nifi"), "--set-file", f"envFile.raw={settings}"], + text=True, capture_output=True, timeout=30, + ) + self.assertNotEqual(result.returncode, 0) + self.assertIn("must be smaller", result.stderr) + + def test_heap_validation_handles_units_and_invalid_combinations(self) -> None: + values = {"NIFI_JVM_HEAP_INIT": "768m", "NIFI_JVM_HEAP_MAX": "1024m", "NIFI_DOCKER_RAM": "2GiB"} + self.assertEqual(resource_errors(values), []) + for changes in ( + {"NIFI_JVM_HEAP_INIT": "2g"}, {"NIFI_JVM_HEAP_MAX": "2g"}, + {"NIFI_JVM_HEAP_MAX": "1.5g"}, {"NIFI_DOCKER_RAM": "unlimited"}, {"NIFI_JVM_HEAP_INIT": "0"}, + ): + with self.subTest(changes=changes): + self.assertTrue(resource_errors({**values, **changes})) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tests/test_opensearch_ai_init.py b/scripts/tests/test_opensearch_ai_init.py index 2c3590885..94f40e9e0 100644 --- a/scripts/tests/test_opensearch_ai_init.py +++ b/scripts/tests/test_opensearch_ai_init.py @@ -93,12 +93,16 @@ def setUp(self): self.root = Path(self.temp.name) (self.root / "scripts").mkdir() (self.root / "deploy").mkdir() + (self.root / "security/env").mkdir(parents=True) + (self.root / "security/env/users_elasticsearch.env").write_text("ELASTIC_USER=admin\n") + (self.root / "deploy/elasticsearch.env").write_text("") + shutil.copy(PROJECT_DIR / "deploy/config_files.sh", self.root / "deploy/config_files.sh") self.script = self.root / "scripts/opensearch-ai-init.sh" shutil.copy(PROJECT_DIR / "scripts/opensearch-ai-init.sh", self.script) (self.root / "deploy/ollama.env").write_text( "OLLAMA_MODEL=file-model\nOPENSEARCH_AI_CHAT_ENDPOINT=ollama-openai-proxy:11435\n" ) - shutil.copy(PROJECT_DIR / "deploy/sglang.env", self.root / "deploy/sglang.env") + (self.root / "deploy/sglang.env").write_text("SGLANG_MODEL=\n") docker = self.root / "docker" docker.write_text(FAKE_DOCKER) docker.chmod(0o755) @@ -114,7 +118,7 @@ def setUp(self): })) self.env = { key: value for key, value in os.environ.items() - if not key.startswith(("OPENSEARCH_AI_", "SGLANG_", "OLLAMA_")) + if not key.startswith(("OPENSEARCH_AI_", "SGLANG_", "OLLAMA_", "COGSTACK_")) } self.env.update({ "PATH": f"{self.root}:{os.environ['PATH']}", diff --git a/security/scripts/create_es_native_certs.sh b/security/scripts/create_es_native_certs.sh index 9270712c3..a23492c07 100755 --- a/security/scripts/create_es_native_certs.sh +++ b/security/scripts/create_es_native_certs.sh @@ -46,8 +46,6 @@ cleanup_es_native_artifacts() { if [ ! -d "../certificates/elastic/elasticsearch" ]; then echo "====================================== CREATE_ES_NATIVE_CERTS ===============================" - source ../env/certificates_elasticsearch.env - source ../env/users_elasticsearch.env echo "Removing previous cert container and volume if existent...." cleanup_es_native_artifacts || true diff --git a/security/scripts/create_es_native_credentials.sh b/security/scripts/create_es_native_credentials.sh index 0709e81c4..daab08802 100755 --- a/security/scripts/create_es_native_credentials.sh +++ b/security/scripts/create_es_native_credentials.sh @@ -32,15 +32,20 @@ set -uo pipefail +# shellcheck disable=SC1090,SC1091 +source "$(dirname "${BASH_SOURCE[0]}")/load_config.sh" || exit 1 + -SECURITY_ENV_FOLDER="../env/" SECURITY_CERTIFICATES_FOLDER="../certificates/" ES_CERTIFICATES_FOLDER="${SECURITY_CERTIFICATES_FOLDER}elastic/elasticsearch" # Load required env vars -source "${SECURITY_ENV_FOLDER}"certificates_elasticsearch.env -source "${SECURITY_ENV_FOLDER}"certificates_general.env -source "${SECURITY_ENV_FOLDER}"users_elasticsearch.env +# shellcheck disable=SC1090 +source "$(cogstack_env_file security/env/certificates_elasticsearch.env)" || exit 1 +# shellcheck disable=SC1090 +source "$(cogstack_env_file security/env/certificates_general.env)" || exit 1 +# shellcheck disable=SC1090 +source "$(cogstack_env_file security/env/users_elasticsearch.env)" || exit 1 ES_CA_CERT="${ES_CERTIFICATES_FOLDER}/elastic-stack-ca.crt.pem" diff --git a/security/scripts/create_gitea_certs.sh b/security/scripts/create_gitea_certs.sh index 21ea353a8..e2c029e47 100755 --- a/security/scripts/create_gitea_certs.sh +++ b/security/scripts/create_gitea_certs.sh @@ -4,6 +4,9 @@ set -euo pipefail +# shellcheck disable=SC1090,SC1091 +source "$(dirname "${BASH_SOURCE[0]}")/load_config.sh" || exit 1 + SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" SECURITY_DIR="${SECURITY_DIR_OVERRIDE:-$(cd -- "${SCRIPT_DIR}/.." && pwd)}" SECURITY_ENV_DIR="${SECURITY_DIR}/env" @@ -12,8 +15,8 @@ ROOT_CERTIFICATES_DIR="${CERTIFICATES_DIR}/root" GITEA_CERTIFICATES_DIR="${CERTIFICATES_DIR}/gitea" EXT_FILE="${SECURITY_DIR}/templates/ssl-extensions-x509.cnf" -# shellcheck disable=SC1091 -source "${SECURITY_ENV_DIR}/certificates_general.env" +# shellcheck disable=SC1090,SC1091 +source "$(cogstack_env_file security/env/certificates_general.env "${SECURITY_ENV_DIR}/certificates_general.env")" || exit 1 : "${ROOT_CERTIFICATE_NAME:?Must be set in certificates_general.env}" : "${ROOT_CERTIFICATE_TIME_VAILIDITY_IN_DAYS:?Must be set in certificates_general.env}" diff --git a/security/scripts/create_keystore.sh b/security/scripts/create_keystore.sh index 6a31b1d59..48fd387bd 100755 --- a/security/scripts/create_keystore.sh +++ b/security/scripts/create_keystore.sh @@ -24,13 +24,16 @@ set -euo pipefail +# shellcheck disable=SC1090,SC1091 +source "$(dirname "${BASH_SOURCE[0]}")/load_config.sh" || exit 1 + SECURITY_CERTIFICATES_FOLDER="../certificates/" -SECURITY_ENV_FOLDER="../env/" ROOT_CERTIFICATES_FOLDER="${SECURITY_CERTIFICATES_FOLDER}root/" -source "${SECURITY_ENV_FOLDER}certificates_general.env" +# shellcheck disable=SC1090 +source "$(cogstack_env_file security/env/certificates_general.env)" || exit 1 # Validate required variables : "${ROOT_CERTIFICATE_NAME:?ROOT_CERTIFICATE_NAME must be set in certificates_general.env}" diff --git a/security/scripts/create_nifi_certs.sh b/security/scripts/create_nifi_certs.sh index b1f36f249..69f857a17 100755 --- a/security/scripts/create_nifi_certs.sh +++ b/security/scripts/create_nifi_certs.sh @@ -2,18 +2,20 @@ set -euo pipefail +# shellcheck disable=SC1090,SC1091 +source "$(dirname "${BASH_SOURCE[0]}")/load_config.sh" || exit 1 + SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" SECURITY_TEMPLATES_FOLDER="${SCRIPT_DIR}/../templates/" SECURITY_CERTIFICATES_FOLDER="${SCRIPT_DIR}/../certificates/" -SECURITY_ENV_FOLDER="${SCRIPT_DIR}/../env/" ROOT_CERTIFICATES_FOLDER="${SECURITY_CERTIFICATES_FOLDER}root/" NIFI_CERTIFICATES_FOLDER="${SECURITY_CERTIFICATES_FOLDER}nifi/" # These paths are resolved dynamically from this script's location. # shellcheck disable=SC1090 -source "${SECURITY_ENV_FOLDER}certificates_general.env" +source "$(cogstack_env_file security/env/certificates_general.env)" || exit 1 # shellcheck disable=SC1090 -source "${SECURITY_ENV_FOLDER}certificates_nifi.env" +source "$(cogstack_env_file security/env/certificates_nifi.env)" || exit 1 : "${ROOT_CERTIFICATE_NAME:?Must be set in certificates_general.env}" : "${NIFI_CERTIFICATE_TIME_VAILIDITY_IN_DAYS:?Must be set in certificates_nifi.env}" diff --git a/security/scripts/create_opensearch_client_admin_certs.sh b/security/scripts/create_opensearch_client_admin_certs.sh index 8d1e8f82e..6c10cde68 100755 --- a/security/scripts/create_opensearch_client_admin_certs.sh +++ b/security/scripts/create_opensearch_client_admin_certs.sh @@ -18,17 +18,21 @@ set -euo pipefail +# shellcheck disable=SC1090,SC1091 +source "$(dirname "${BASH_SOURCE[0]}")/load_config.sh" || exit 1 + SECURITY_TEMPLATES_FOLDER="../templates/" SECURITY_CERTIFICATES_FOLDER="../certificates/" -SECURITY_ENV_FOLDER="../env/" ROOT_CERTIFICATES_FOLDER="${SECURITY_CERTIFICATES_FOLDER}root/" OPENSEARCH_ES_CERTIFICATES_FOLDER="${SECURITY_CERTIFICATES_FOLDER}elastic/opensearch/" -source "${SECURITY_ENV_FOLDER}/certificates_general.env" -source "${SECURITY_ENV_FOLDER}/certificates_elasticsearch.env" +# shellcheck disable=SC1090 +source "$(cogstack_env_file security/env/certificates_general.env)" || exit 1 +# shellcheck disable=SC1090 +source "$(cogstack_env_file security/env/certificates_elasticsearch.env)" || exit 1 # Validate required env vars : "${ROOT_CERTIFICATE_NAME:?Must be set in certificates_general.env}" diff --git a/security/scripts/create_opensearch_internal_passwords.sh b/security/scripts/create_opensearch_internal_passwords.sh index 2546be923..3df3c1969 100755 --- a/security/scripts/create_opensearch_internal_passwords.sh +++ b/security/scripts/create_opensearch_internal_passwords.sh @@ -20,13 +20,18 @@ set -euo pipefail +# shellcheck disable=SC1090,SC1091 +source "$(dirname "${BASH_SOURCE[0]}")/load_config.sh" || exit 1 + -SECURITY_ENV_FOLDER="../env/" # required env var files -source ../../deploy/elasticsearch.env -source "${SECURITY_ENV_FOLDER}certificates_elasticsearch.env" -source "${SECURITY_ENV_FOLDER}users_elasticsearch.env" +# shellcheck disable=SC1090 +source "$(cogstack_env_file deploy/elasticsearch.env)" || exit 1 +# shellcheck disable=SC1090 +source "$(cogstack_env_file security/env/certificates_elasticsearch.env)" || exit 1 +# shellcheck disable=SC1090 +source "$(cogstack_env_file security/env/users_elasticsearch.env)" || exit 1 # === Validate input === if [[ $# -lt 1 || $# -gt 2 ]]; then diff --git a/security/scripts/create_opensearch_node_cert.sh b/security/scripts/create_opensearch_node_cert.sh index 4b0685b7b..33fcd5cf0 100755 --- a/security/scripts/create_opensearch_node_cert.sh +++ b/security/scripts/create_opensearch_node_cert.sh @@ -25,17 +25,22 @@ set -euo pipefail +# shellcheck disable=SC1090,SC1091 +source "$(dirname "${BASH_SOURCE[0]}")/load_config.sh" || exit 1 + SECURITY_TEMPLATES_FOLDER="../templates/" SECURITY_CERTIFICATES_FOLDER="../certificates/" -SECURITY_ENV_FOLDER="../env/" ROOT_CERTIFICATES_FOLDER="${SECURITY_CERTIFICATES_FOLDER}root/" # required env var files -source "${SECURITY_ENV_FOLDER}certificates_general.env" -source "${SECURITY_ENV_FOLDER}certificates_elasticsearch.env" -source "${SECURITY_ENV_FOLDER}users_elasticsearch.env" +# shellcheck disable=SC1090 +source "$(cogstack_env_file security/env/certificates_general.env)" || exit 1 +# shellcheck disable=SC1090 +source "$(cogstack_env_file security/env/certificates_elasticsearch.env)" || exit 1 +# shellcheck disable=SC1090 +source "$(cogstack_env_file security/env/users_elasticsearch.env)" || exit 1 # === Required Input === if [[ "$#" -lt 1 ]]; then diff --git a/security/scripts/create_opensearch_users.sh b/security/scripts/create_opensearch_users.sh index b850b806b..7cbc7da7e 100755 --- a/security/scripts/create_opensearch_users.sh +++ b/security/scripts/create_opensearch_users.sh @@ -5,11 +5,12 @@ set -euo pipefail +# shellcheck disable=SC1090,SC1091 +source "$(dirname "${BASH_SOURCE[0]}")/load_config.sh" || exit 1 + SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" SECURITY_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" -REPO_ROOT="$(cd "${SECURITY_DIR}/.." && pwd)" -SECURITY_ENV_DIR="${SECURITY_DIR}/env" DEFAULT_CA_CERT="${SECURITY_DIR}/certificates/elastic/opensearch/elastic-stack-ca.crt.pem" usage() { @@ -50,10 +51,10 @@ json_escape() { } source_env_files() { - # shellcheck disable=SC1091 - source "${REPO_ROOT}/deploy/elasticsearch.env" - # shellcheck disable=SC1091 - source "${SECURITY_ENV_DIR}/users_elasticsearch.env" + # shellcheck disable=SC1090,SC1091 + source "$(cogstack_env_file deploy/elasticsearch.env)" || exit 1 + # shellcheck disable=SC1090,SC1091 + source "$(cogstack_env_file security/env/users_elasticsearch.env)" || exit 1 } OPENSEARCH_HOST="" diff --git a/security/scripts/create_root_ca_cert.sh b/security/scripts/create_root_ca_cert.sh index 167bbbab1..0b7adad38 100755 --- a/security/scripts/create_root_ca_cert.sh +++ b/security/scripts/create_root_ca_cert.sh @@ -22,14 +22,16 @@ set -euo pipefail +# shellcheck disable=SC1090,SC1091 +source "$(dirname "${BASH_SOURCE[0]}")/load_config.sh" || exit 1 + SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" SECURITY_TEMPLATES_FOLDER="${SCRIPT_DIR}/../templates/" SECURITY_CERTIFICATES_FOLDER="${SCRIPT_DIR}/../certificates/" -SECURITY_ENV_FOLDER="${SCRIPT_DIR}/../env/" # The path is resolved dynamically from this script's location. # shellcheck disable=SC1090 -source "${SECURITY_ENV_FOLDER}certificates_general.env" +source "$(cogstack_env_file security/env/certificates_general.env)" || exit 1 # === Validate required env vars : "${ROOT_CERTIFICATE_NAME:?Must be set in certificates_general.env}" diff --git a/security/scripts/load_config.sh b/security/scripts/load_config.sh new file mode 100755 index 000000000..a5f1e1da7 --- /dev/null +++ b/security/scripts/load_config.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +# Source this helper before selecting deployment-specific security env files. + +COGSTACK_SECURITY_CONFIG_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +# shellcheck disable=SC1091 +source "$COGSTACK_SECURITY_CONFIG_ROOT/deploy/config_files.sh" || return 1 +cogstack_config_init "$COGSTACK_SECURITY_CONFIG_ROOT" || return 1 diff --git a/security/scripts/update_opensearch_users.sh b/security/scripts/update_opensearch_users.sh index 7bda8a409..0ffcea3ca 100755 --- a/security/scripts/update_opensearch_users.sh +++ b/security/scripts/update_opensearch_users.sh @@ -25,9 +25,11 @@ set -euo pipefail +# shellcheck disable=SC1090,SC1091 +source "$(dirname "${BASH_SOURCE[0]}")/load_config.sh" || exit 1 + SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" SECURITY_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" -REPO_ROOT="$(cd "${SECURITY_DIR}/.." && pwd)" SECURITY_ENV_FOLDER="${SECURITY_DIR}/env" INTERNAL_USERS_YML="${SECURITY_DIR}/es_roles/opensearch/internal_users.yml" @@ -47,14 +49,14 @@ USAGE } source_env_files() { - # shellcheck disable=SC1091 - source "${REPO_ROOT}/deploy/elasticsearch.env" - # shellcheck disable=SC1091 - source "${SECURITY_ENV_FOLDER}/certificates_elasticsearch.env" - # shellcheck disable=SC1091 - source "${SECURITY_ENV_FOLDER}/certificates_general.env" - # shellcheck disable=SC1091 - source "${SECURITY_ENV_FOLDER}/users_elasticsearch.env" + # shellcheck disable=SC1090,SC1091 + source "$(cogstack_env_file deploy/elasticsearch.env)" || exit 1 + # shellcheck disable=SC1090,SC1091 + source "$(cogstack_env_file security/env/certificates_elasticsearch.env)" || exit 1 + # shellcheck disable=SC1090,SC1091 + source "$(cogstack_env_file security/env/certificates_general.env)" || exit 1 + # shellcheck disable=SC1090,SC1091 + source "$(cogstack_env_file security/env/users_elasticsearch.env)" || exit 1 } configure_opensearch_paths() { From 486a080e17ce6d2c0ee1d8d63e7e75c5607d6385 Mon Sep 17 00:00:00 2001 From: vladd-bit Date: Thu, 29 Oct 2026 13:11:12 +0000 Subject: [PATCH 12/28] Script perms update. --- scripts/tests/test_opensearch_ai_init.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 scripts/tests/test_opensearch_ai_init.py diff --git a/scripts/tests/test_opensearch_ai_init.py b/scripts/tests/test_opensearch_ai_init.py old mode 100644 new mode 100755 From ffc76c173535ad9741d15edcdb96add5749e7f02 Mon Sep 17 00:00:00 2001 From: vladd-bit Date: Thu, 29 Oct 2026 14:13:35 +0000 Subject: [PATCH 13/28] Deploy: synced Helm with docker opensearch v. --- .github/workflows/compose_config.yml | 29 ++ deploy/Makefile | 23 +- deploy/charts/opensearch/Chart.yaml | 4 +- deploy/charts/opensearch/README.md | 31 +++ .../templates/deployment-dashboards.yaml | 8 - .../templates/secret-credentials.yaml | 14 +- .../charts/opensearch/templates/validate.yaml | 22 ++ deploy/charts/opensearch/values.yaml | 12 +- docs/deploy/deployment.md | 8 +- docs/deploy/reproducible.md | 27 +- docs/security/elasticsearch_opensearch.md | 44 ++- scripts/tests/check_deployment_images.py | 5 - scripts/tests/check_opensearch_settings.py | 47 ++++ scripts/tests/smoke_elastic_services.sh | 38 ++- scripts/tests/test_opensearch_deployment.py | 261 ++++++++++++++++++ security/env/certificates_elasticsearch.env | 4 +- security/env/users_elasticsearch.env | 8 +- .../create_opensearch_client_admin_certs.sh | 11 +- .../scripts/create_opensearch_node_cert.sh | 10 +- services/kibana/config/opensearch.yml | 11 +- 20 files changed, 551 insertions(+), 66 deletions(-) create mode 100644 deploy/charts/opensearch/templates/validate.yaml create mode 100755 scripts/tests/check_opensearch_settings.py create mode 100755 scripts/tests/test_opensearch_deployment.py diff --git a/.github/workflows/compose_config.yml b/.github/workflows/compose_config.yml index dfcc31230..3d8fa39cf 100644 --- a/.github/workflows/compose_config.yml +++ b/.github/workflows/compose_config.yml @@ -12,6 +12,12 @@ on: - "scripts/opensearch-ai-init.sh" - "scripts/tests/check_nifi_resources.py" - "scripts/tests/test_configuration.py" + - "scripts/tests/check_opensearch_settings.py" + - "scripts/tests/test_opensearch_deployment.py" + - "services/elasticsearch/**" + - "services/kibana/config/opensearch.yml" + - "security/templates/**" + - "security/es_roles/opensearch/**" - "scripts/tests/test_opensearch_ai_init.py" - "deploy/services*.yml" - "deploy/charts/**" @@ -33,6 +39,12 @@ on: - "scripts/opensearch-ai-init.sh" - "scripts/tests/check_nifi_resources.py" - "scripts/tests/test_configuration.py" + - "scripts/tests/check_opensearch_settings.py" + - "scripts/tests/test_opensearch_deployment.py" + - "services/elasticsearch/**" + - "services/kibana/config/opensearch.yml" + - "security/templates/**" + - "security/es_roles/opensearch/**" - "scripts/tests/test_opensearch_ai_init.py" - "deploy/services*.yml" - "deploy/charts/**" @@ -60,6 +72,20 @@ jobs: steps: - uses: actions/checkout@v7 + - name: Set up Python + uses: actions/setup-python@v7 + with: + python-version: "3.11" + + - name: Set up Java for certificate generation tests + uses: actions/setup-java@v6 + with: + distribution: temurin + java-version: '21' + + - name: Install deployment test dependency + run: python3 -m pip install PyYAML==6.0.3 + - name: Set up Helm uses: azure/setup-helm@v5.0.1 with: @@ -81,3 +107,6 @@ jobs: - name: Test OpenSearch AI configuration run: python3 -B -m unittest discover -s scripts/tests -p test_opensearch_ai_init.py -v + + - name: Test OpenSearch Compose and Helm compatibility + run: python3 -B -m unittest discover -s scripts/tests -p test_opensearch_deployment.py -v diff --git a/deploy/Makefile b/deploy/Makefile index 1e420825f..edcab1cc5 100644 --- a/deploy/Makefile +++ b/deploy/Makefile @@ -326,10 +326,13 @@ _check-remote-params: check-nifi-resources: ## Validate NiFi heap sizes against the container memory limit @$(WITH_ENV) python3 ../scripts/tests/check_nifi_resources.py -check-config: check-nifi-resources ## Validate resolved settings and both core Compose files - @$(WITH_ENV) docker compose -f services.yml config --quiet && docker compose -f services.dev.yml config --quiet +check-opensearch-settings: ## Validate search credentials, encryption keys, and image versions + @$(WITH_ENV) python3 ../scripts/tests/check_opensearch_settings.py -.PHONY: check-nifi-resources check-config +check-config: check-nifi-resources check-opensearch-settings ## Validate resolved settings and both core Compose files + @$(WITH_ENV) docker compose --profile "*" -f services.yml config --quiet && docker compose --profile "*" -f services.dev.yml config --quiet + +.PHONY: check-nifi-resources check-opensearch-settings check-config # start services @@ -352,19 +355,19 @@ build-nifi-image: ## Build the local NiFi image without starting services --file ../nifi/Dockerfile \ .. -start-elastic: ## Start elasticsearch-1, elasticsearch-2 and kibana +start-elastic: check-opensearch-settings ## Start elasticsearch-1, elasticsearch-2 and kibana $(WITH_ENV) docker compose -f services.yml $(DC_START_CMD) elasticsearch-1 elasticsearch-2 kibana -start-elastic-cluster: ## Start full 3-node Elasticsearch cluster +start-elastic-cluster: check-opensearch-settings ## Start full 3-node Elasticsearch cluster $(WITH_ENV) docker compose -f services.yml $(DC_START_CMD) elasticsearch-1 elasticsearch-2 elasticsearch-3 -start-elastic-1: +start-elastic-1: check-opensearch-settings $(WITH_ENV) docker compose -f services.yml $(DC_START_CMD) elasticsearch-1 -start-elastic-2: +start-elastic-2: check-opensearch-settings $(WITH_ENV) docker compose -f services.yml $(DC_START_CMD) elasticsearch-2 -start-elastic-3: +start-elastic-3: check-opensearch-settings $(WITH_ENV) docker compose -f services.yml $(DC_START_CMD) elasticsearch-3 start-metricbeat-1: @@ -385,7 +388,7 @@ start-filebeat-2: start-filebeat-3: $(WITH_ENV) docker compose -f services.yml $(DC_START_CMD) filebeat-3 -start-kibana: +start-kibana: check-opensearch-settings ## Start Kibana or OpenSearch Dashboards $(WITH_ENV) docker compose -f services.yml $(DC_START_CMD) kibana start-ollama: ## Start the Ollama inference service @@ -459,7 +462,7 @@ start-data-infra: start-nifi start-elastic start-samples ## Start core data infr start-all: start-data-infra start-medcat-service start-ocr-services ## Start all major services -.PHONY: start-all start-data-infra start-nifi start-nifi-dev start-nifi-dev-build build-nifi-image start-elastic start-ollama start-ollama-openai-proxy pull-ollama-model setup-ollama opensearch-ai-init opensearch-ai-init-sglang start-openwebui restart-openwebui status-openwebui logs-openwebui start-samples start-clickhouse status-clickhouse logs-clickhouse start-jupyter +.PHONY: start-kibana start-all start-data-infra start-nifi start-nifi-dev start-nifi-dev-build build-nifi-image start-elastic start-ollama start-ollama-openai-proxy pull-ollama-model setup-ollama opensearch-ai-init opensearch-ai-init-sglang start-openwebui restart-openwebui status-openwebui logs-openwebui start-samples start-clickhouse status-clickhouse logs-clickhouse start-jupyter # stop services diff --git a/deploy/charts/opensearch/Chart.yaml b/deploy/charts/opensearch/Chart.yaml index fd6e92311..ae05911e2 100644 --- a/deploy/charts/opensearch/Chart.yaml +++ b/deploy/charts/opensearch/Chart.yaml @@ -2,5 +2,5 @@ apiVersion: v2 name: cogstack-opensearch description: Helm chart for deploying CogStack OpenSearch and OpenSearch Dashboards type: application -version: 0.1.0 -appVersion: "3.5.0" +version: 0.1.1 +appVersion: "3.9.0" diff --git a/deploy/charts/opensearch/README.md b/deploy/charts/opensearch/README.md index afd81fcb5..69d0aef54 100644 --- a/deploy/charts/opensearch/README.md +++ b/deploy/charts/opensearch/README.md @@ -23,6 +23,23 @@ Helm chart for deploying OpenSearch and/or OpenSearch Dashboards using the CogSt - `OPENSEARCH_COMPLIANCE_SALT` and `OPENSEARCH_DATASOURCE_MASTER_KEY` when `opensearch.enabled=true` - `KIBANA_USER` and `KIBANA_PASSWORD` when `dashboards.enabled=true` +Compose and this chart use the same OpenSearch and Dashboards 3.9.0 image +digests. CI compares these selections, the shared configuration, generated +credential Secrets, and real certificate SANs. A normal render uses the public +development values in `security/env/users_elasticsearch.env`. + +The compliance salt requires at least 16 ASCII characters and the datasource +key requires 16, 24, or 32 ASCII characters. Keep them identical on every node +and stable across upgrades. Generated Secrets are validated at render time; +when using an existing Secret, provide all required keys yourself. + +This deployment disables the demo installer. `OPENSEARCH_INITIAL_ADMIN_PASSWORD` +does not rotate the users in the supplied security configuration. The bundled +development logins are `admin` / `admin` and service account `kibanaserver` / +`kibanaserver`. Keep deployment passwords in sync with `internal_users.yml` and +the running security index using the +[user rotation procedure](../../../docs/security/elasticsearch_opensearch.md#update_opensearch_userssh-reference). + ## Required certificate secrets Set these in `values.yaml` for enabled components: @@ -50,9 +67,23 @@ Repo-aligned certificate source paths: - `security/certificates/elastic/opensearch/elasticsearch/elasticsearch-3/elasticsearch-3.crt` - `security/certificates/elastic/opensearch/elasticsearch/elasticsearch-3/elasticsearch-3.key` - Dashboards certs: + - `security/certificates/elastic/opensearch/elastic-stack-ca.crt.pem` - `security/certificates/elastic/opensearch/es_kibana_client.pem` - `security/certificates/elastic/opensearch/es_kibana_client.key` +Dashboards uses its configured service account and full CA/hostname +verification. It needs only the CA and HTTPS server certificate/key, and can +use a separate Secret through `certificates.dashboardsSecretName`. + +Generate certificates with `make -C deploy init-security-opensearch`. The +default SANs cover release `cogstack-opensearch` in namespace `cogstack`, +including `cogstack-opensearch-client.cogstack.svc` and +`cogstack-opensearch-dashboards.cogstack.svc` and their `.cluster.local` forms. +For other names, change the selected `certificates_elasticsearch.env` before +generation. Complete existing certificate sets are preserved: reissue any leaf +missing these SANs with the existing CA, update the TLS Secrets, and restart +the affected pods. + Example secret creation from the repo layout: ```bash diff --git a/deploy/charts/opensearch/templates/deployment-dashboards.yaml b/deploy/charts/opensearch/templates/deployment-dashboards.yaml index 806cdb40e..0d6444198 100644 --- a/deploy/charts/opensearch/templates/deployment-dashboards.yaml +++ b/deploy/charts/opensearch/templates/deployment-dashboards.yaml @@ -86,14 +86,6 @@ spec: mountPath: /certificates/elastic/opensearch/elastic-stack-ca.crt.pem subPath: {{ .Values.certificates.dashboardsFiles.rootCA }} readOnly: true - - name: dashboards-certs - mountPath: /certificates/elastic/opensearch/admin.crt - subPath: {{ .Values.certificates.dashboardsFiles.adminCert }} - readOnly: true - - name: dashboards-certs - mountPath: /certificates/elastic/opensearch/admin.key.pem - subPath: {{ .Values.certificates.dashboardsFiles.adminKey }} - readOnly: true - name: dashboards-certs mountPath: /certificates/elastic/opensearch/es_kibana_client.pem subPath: {{ $dashboardsServerCert }} diff --git a/deploy/charts/opensearch/templates/secret-credentials.yaml b/deploy/charts/opensearch/templates/secret-credentials.yaml index 6a5facc6c..c225cdc7c 100644 --- a/deploy/charts/opensearch/templates/secret-credentials.yaml +++ b/deploy/charts/opensearch/templates/secret-credentials.yaml @@ -20,6 +20,14 @@ {{- if hasKey $usersData "KIBANA_PASSWORD" -}} {{- $kibanaPassword = index $usersData "KIBANA_PASSWORD" -}} {{- end -}} +{{- if .Values.opensearch.enabled -}} +{{- if or (lt (len $complianceSalt) 16) (regexMatch "[^\\x00-\\x7F]" $complianceSalt) -}} +{{- fail "OPENSEARCH_COMPLIANCE_SALT must contain at least 16 ASCII characters" -}} +{{- end -}} +{{- if or (not (has (len $datasourceMasterKey) (list 16 24 32))) (regexMatch "[^\\x00-\\x7F]" $datasourceMasterKey) -}} +{{- fail "OPENSEARCH_DATASOURCE_MASTER_KEY must contain 16, 24, or 32 ASCII characters" -}} +{{- end -}} +{{- end -}} apiVersion: v1 kind: Secret metadata: @@ -29,12 +37,12 @@ metadata: type: Opaque stringData: {{- if .Values.opensearch.enabled }} - OPENSEARCH_INITIAL_ADMIN_PASSWORD: {{ $adminPassword | quote }} + OPENSEARCH_INITIAL_ADMIN_PASSWORD: {{ required "OPENSEARCH_INITIAL_ADMIN_PASSWORD must not be empty" $adminPassword | quote }} OPENSEARCH_COMPLIANCE_SALT: {{ required "Set OPENSEARCH_COMPLIANCE_SALT in usersEnvFile.raw or credentials.complianceSalt" $complianceSalt | quote }} OPENSEARCH_DATASOURCE_MASTER_KEY: {{ required "Set OPENSEARCH_DATASOURCE_MASTER_KEY in usersEnvFile.raw or credentials.datasourceMasterKey" $datasourceMasterKey | quote }} {{- end }} {{- if .Values.dashboards.enabled }} - KIBANA_USER: {{ $kibanaUser | quote }} - KIBANA_PASSWORD: {{ $kibanaPassword | quote }} + KIBANA_USER: {{ required "KIBANA_USER must not be empty" $kibanaUser | quote }} + KIBANA_PASSWORD: {{ required "KIBANA_PASSWORD must not be empty" $kibanaPassword | quote }} {{- end }} {{- end }} diff --git a/deploy/charts/opensearch/templates/validate.yaml b/deploy/charts/opensearch/templates/validate.yaml new file mode 100644 index 000000000..01b4151ca --- /dev/null +++ b/deploy/charts/opensearch/templates/validate.yaml @@ -0,0 +1,22 @@ +{{- if or .Values.opensearch.enabled .Values.dashboards.enabled -}} +{{- if and (not .Values.credentials.create) (not .Values.credentials.existingSecret) -}} +{{- fail "credentials.existingSecret is required when credentials.create=false" -}} +{{- end -}} +{{- end -}} +{{- if .Values.opensearch.enabled -}} +{{- $_ := required "certificates.opensearchSecretName is required" .Values.certificates.opensearchSecretName -}} +{{- if lt (int .Values.opensearch.replicas) 1 -}} +{{- fail "opensearch.replicas must be at least 1" -}} +{{- end -}} +{{- $nodes := .Values.certificates.opensearchNodeFiles | default (list) -}} +{{- if and (gt (len $nodes) 0) (lt (len $nodes) (int .Values.opensearch.replicas)) -}} +{{- fail "certificates.opensearchNodeFiles must provide a certificate/key pair for every OpenSearch replica" -}} +{{- end -}} +{{- range $node := $nodes -}} +{{- $_ := required "Every opensearchNodeFiles entry requires cert" $node.cert -}} +{{- $_ := required "Every opensearchNodeFiles entry requires key" $node.key -}} +{{- end -}} +{{- end -}} +{{- if .Values.dashboards.enabled -}} +{{- $_ := required "certificates.dashboardsSecretName is required" .Values.certificates.dashboardsSecretName -}} +{{- end -}} diff --git a/deploy/charts/opensearch/values.yaml b/deploy/charts/opensearch/values.yaml index 08aa9d4aa..cdabab675 100644 --- a/deploy/charts/opensearch/values.yaml +++ b/deploy/charts/opensearch/values.yaml @@ -51,8 +51,8 @@ opensearch: replicas: 3 image: repository: opensearchproject/opensearch - tag: "3.5.0" - digest: sha256:dbb01641baadae5104e18acd888bf05e8fdd9af3567fd30624a76ba3e5a31dec + tag: "3.9.0" + digest: sha256:3b55f6b6249cfdb92b52ee59adc8a266203ba9ef9f5d82544fad6f1af2f3609e pullPolicy: IfNotPresent clusterName: elasticsearch-cogstack-cluster javaOpts: "-Xms512m -Xmx512m -Des.failure_store_feature_flag_enabled=true --enable-native-access=ALL-UNNAMED" @@ -128,8 +128,8 @@ dashboards: opensearchHosts: [] image: repository: opensearchproject/opensearch-dashboards - tag: "3.5.0" - digest: sha256:03afc96f2f50a02246294aaa37adffc41270104676eff7c5f377c40ae9dd1ff2 + tag: "3.9.0" + digest: sha256:256d4cea5928868ac09bca642a0ac8285a2b7e4b7fdd45c4ec06e75a5cf3184c pullPolicy: IfNotPresent serverName: cogstack-kibana service: @@ -154,7 +154,7 @@ credentials: create: true existingSecret: "" adminPassword: "Kibanaserver!1" - # Required when usersEnvFile.raw overrides the bundled development values. + # Required for OpenSearch; bundled development values come from usersEnvFile. # Use deployment-specific values or credentials.existingSecret in production. complianceSalt: "" datasourceMasterKey: "" @@ -197,7 +197,5 @@ certificates: dashboardsFiles: # Secret key names for Dashboards cert material. rootCA: elastic-stack-ca.crt.pem - adminCert: admin.crt - adminKey: admin.key.pem serverCert: es_kibana_client.pem serverKey: es_kibana_client.key diff --git a/docs/deploy/deployment.md b/docs/deploy/deployment.md index 940ef3868..f5a536065 100644 --- a/docs/deploy/deployment.md +++ b/docs/deploy/deployment.md @@ -135,6 +135,11 @@ helm upgrade --install cogstack-opensearch ./deploy/charts/opensearch \ > The chart already consumes the shared OpenSearch, Dashboards, and security YAML files automatically from this repo. > The values file is only for cluster-specific overrides such as secret names, storage classes, replicas, and snapshot PVC claims. +Compose and Helm use the same OpenSearch and Dashboards 3.9.0 image digests. +See [compatibility checks](reproducible.md#opensearch-compose-and-helm-compatibility) +and the [credentials and certificate requirements](../security/elasticsearch_opensearch.md). +The Helm Make targets also honor `COGSTACK_CONFIG_DIR`. + ## ⎈ Helm (GitEA / Gitea) GitEA can be deployed with the official Gitea Helm chart. The Makefile adds the @@ -252,8 +257,9 @@ make -C deploy remote-delete-service \ | Command | Description | |------------------------|---------------------------------------------| | `make load-env` | Load all environment variables | -| `make check-config` | Validate selected configuration, NiFi memory, and both core Compose files | +| `make check-config` | Validate selected configuration, NiFi memory, search settings, and both core Compose files | | `make check-nifi-resources` | Check JVM heap sizes against the container memory limit | +| `make check-opensearch-settings` | Check search credentials, encryption keys, and image versions | | `make show-env` | Print environment variables (sorted) | | `make init-security` | Generate missing NiFi, Gitea, and selected search-backend certificates | | `make init-security-root-ca` | Generate the shared root CA if missing | diff --git a/docs/deploy/reproducible.md b/docs/deploy/reproducible.md index 5cff7d085..23e253817 100644 --- a/docs/deploy/reproducible.md +++ b/docs/deploy/reproducible.md @@ -124,9 +124,30 @@ python3 -B -m unittest discover -s scripts/tests -p test_submodule_checkout.py - The image check requires Docker Compose and Helm, but no running Docker daemon or Kubernetes cluster. It validates both core Compose files, their local build -bases, and the three owned Helm charts. Helm image validation supplies an -existing credentials Secret name; deployment-specific credentials and TLS -Secrets must still be provided for an installation. +bases, and the three owned Helm charts. OpenSearch renders its normal generated +credentials Secret from the shared development settings. An installation still +requires TLS Secrets and deployment-specific credentials. + +## OpenSearch Compose and Helm compatibility + +The reference Compose deployment and Helm chart both select OpenSearch and +OpenSearch Dashboards 3.9.0 with matching image digests. When upgrading, update +`deploy/elasticsearch.env`, both images in `deploy/charts/opensearch/values.yaml`, +and the chart's `appVersion` together. + +```bash +make -C deploy check-opensearch-settings +python3 -B -m unittest discover -s scripts/tests -p test_opensearch_deployment.py -v +``` + +The compatibility tests require Helm, Docker Compose, PyYAML, OpenSSL, and Java +`keytool`. They compare image versions and digests, shared configuration, +credential Secret references, profile selection, and TLS mappings. They also +generate temporary certificates and verify the Compose and Kubernetes service +names against a temporary CA. They do not change existing certificates or +start a cluster. CI runs these checks and a separate live Compose smoke test +that verifies password authentication, the running OpenSearch version, and +Dashboards readiness over verified HTTPS. Optional services such as OCR, MedCAT, and JupyterHub use deployment definitions inside their pinned submodules. Image and model versioning within those diff --git a/docs/security/elasticsearch_opensearch.md b/docs/security/elasticsearch_opensearch.md index aa28dc3c6..59d31e1be 100644 --- a/docs/security/elasticsearch_opensearch.md +++ b/docs/security/elasticsearch_opensearch.md @@ -58,7 +58,7 @@ Apply the same configuration to every OpenSearch node and perform a rolling rest The shared OpenSearch configuration reads `plugins.security.compliance.salt` from `OPENSEARCH_COMPLIANCE_SALT` and `plugins.query.datasources.encryption.masterkey` from `OPENSEARCH_DATASOURCE_MASTER_KEY`. All nodes must receive identical values. Compose reads them from `security/env/users_elasticsearch.env`; the Helm chart imports them into the credentials Secret and references that Secret from every OpenSearch pod. -The checked-in values are public development defaults. Supply deployment-specific values through local configuration or an external secret store for production. Generate a 16-character ASCII salt with `openssl rand -hex 8` and a 32-character datasource key with `openssl rand -hex 16`. Preserve both values across restarts and keep a backup of the key: changing it prevents decryption of credentials already stored in SQL/PPL datasource definitions. Changing the salt changes field-masking hashes. See [Field masking](https://docs.opensearch.org/latest/security/access-control/field-masking/#set-the-salt-setting) and [Data sources](https://docs.opensearch.org/latest/dashboards/management/data-sources/). +The checked-in values are public development defaults. The salt requires at least 16 ASCII characters; the datasource key requires 16, 24, or 32 ASCII characters. Supply deployment-specific values through local configuration or an external secret store for production. Generate a 16-character ASCII salt with `openssl rand -hex 8` and a 32-character datasource key with `openssl rand -hex 16`. Preserve both values across restarts and keep a backup of the key: changing it prevents decryption of credentials already stored in SQL/PPL datasource definitions. Changing the salt changes field-masking hashes. See [Field masking](https://docs.opensearch.org/latest/security/access-control/field-masking/#set-the-salt-setting) and [Data sources](https://docs.opensearch.org/latest/dashboards/management/data-sources/). For Helm, supply these keys in `usersEnvFile.raw` together with the existing credential keys, or include them in `credentials.existingSecret`. When overriding `usersEnvFile.raw` without these keys, set `credentials.complianceSalt` and `credentials.datasourceMasterKey` explicitly. Recreate Compose containers one node at a time after changing their environment; `docker compose restart` does not reload `env_file` values. @@ -168,8 +168,8 @@ This produces: | File | Purpose | |------|----------| -| `admin.crt`, `admin.key.pem` | Admin dashboard certificate | -| `es_kibana_client.pem`, `es_kibana_client.key` | Client certificate for Kibana/OpenDashboard | +| `admin.crt`, `admin.key.pem` | Administrative client certificate for security management | +| `es_kibana_client.pem`, `es_kibana_client.key` | HTTPS server certificate for OpenSearch Dashboards | | `*.jks` | Node keystores/truststores for HTTPS and inter-node encryption | The resulting certificates are placed in: @@ -178,6 +178,19 @@ The resulting certificates are placed in: security/certificates/elastic/opensearch/ ``` +The signing scripts preserve the SANs configured in `ES_NODE_SUBJ_ALT_NAMES` and +`ES_CLIENT_SUBJ_ALT_NAMES`. The defaults cover Compose names, localhost, and the +Helm release `cogstack-opensearch` in namespace `cogstack`: node certificates +include `cogstack-opensearch-client.cogstack.svc` and its `.cluster.local` form; +the Dashboards certificate covers the corresponding +`cogstack-opensearch-dashboards` Service. Add service names for a different +release, namespace, cluster domain, or external hostname before generation. + +`init-security-opensearch` preserves complete existing certificate sets. If an +existing leaf certificate lacks these SANs, plan a leaf certificate reissue +using the existing CA, then update the mounted files or Kubernetes TLS Secret +and restart the affected services. Keep the existing CA and data volumes. + --- ### 📁 Kibana / OpenDashboard certificates @@ -185,16 +198,39 @@ security/certificates/elastic/opensearch/ | Platform | Required Certificates | Source Folder | |-----------|----------------------|----------------| | **Kibana** | `elasticsearch-{1,2,3}.crt`, `elasticsearch-{1,2,3}.key`, `elastic-stack-ca.crt.pem` | `security/certificates/elastic/elasticsearch/` | -| **OpenDashboard (OpenSearch)** | `admin.crt`, `admin.key.pem`, `es_kibana_client.pem`, `es_kibana_client.key`, `elastic-stack-ca.crt.pem` | `security/certificates/elastic/opensearch/` | +| **OpenDashboard (OpenSearch)** | `es_kibana_client.pem`, `es_kibana_client.key`, `elastic-stack-ca.crt.pem` | `security/certificates/elastic/opensearch/` | All certificate references in `services/kibana/config/opensearch.yml` or `services.yml` must point to these locations. +Dashboards authenticates to OpenSearch with `KIBANA_USER` and `KIBANA_PASSWORD` +and verifies the server CA and hostname. Its outgoing connection does not use +the administrative client certificate. The Helm chart can use a separate +`certificates.dashboardsSecretName` containing only the three Dashboards files. + --- ### 🔐 Users and roles #### OpenSearch +The bundled development admin login is `admin` / `admin`; the Dashboards service +account is `kibanaserver` / `kibanaserver`. These match the hashes in +`security/es_roles/opensearch/internal_users.yml`. `ELASTIC_USER` defaults to +`admin`; select `elastic` when using native Elasticsearch. + +`OPENSEARCH_INITIAL_ADMIN_PASSWORD` is an input to the upstream demo installer, +which this deployment disables. Changing it does not change these logins. +Password values must match the internal-user hashes and the running security +index. Use the rotation procedure below to update both; changing an environment +file or Helm Secret alone does not rotate internal users. See the upstream +[password configuration](https://docs.opensearch.org/latest/security/configuration/passwords/). + +Run `make -C deploy check-opensearch-settings` before starting services. It +validates required credentials, key lengths, and matching image release tags +without printing secret values. Helm validates generated credentials and TLS +Secret mappings during rendering; external Secret contents must satisfy the +same requirements. + 1. Edit the password values in `security/env/users_elasticsearch.env` and ensure every required user is present in `security/es_roles/opensearch/internal_users.yml`. 2. With OpenSearch running, validate and then apply hashes for all internal users: diff --git a/scripts/tests/check_deployment_images.py b/scripts/tests/check_deployment_images.py index d75a66369..62ca1661d 100755 --- a/scripts/tests/check_deployment_images.py +++ b/scripts/tests/check_deployment_images.py @@ -72,11 +72,6 @@ def helm_errors() -> list[str]: "helm", "template", f"check-{chart}", str(ROOT / "deploy/charts" / chart), "-f", str(ROOT / "deploy/helm" / f"{chart}.values.yaml"), ] - if chart == "opensearch": - # Image rendering only: deployment credentials are supplied externally. - command.extend([ - "--set", "credentials.create=false", "--set", "credentials.existingSecret=check-credentials", - ]) result = subprocess.run(command, text=True, capture_output=True, check=True, timeout=60) images = re.findall(r"^\s*(?:image|imageName):\s*(\S+)", result.stdout, re.MULTILINE) if not images: diff --git a/scripts/tests/check_opensearch_settings.py b/scripts/tests/check_opensearch_settings.py new file mode 100755 index 000000000..9d1f04c27 --- /dev/null +++ b/scripts/tests/check_opensearch_settings.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +"""Validate shared OpenSearch credentials and image selections before startup.""" + +from __future__ import annotations + +import os +import re +import sys +from collections.abc import Mapping + + +def setting_errors(values: Mapping[str, str]) -> list[str]: + if values.get("ELASTICSEARCH_VERSION") == "elasticsearch": + return [] + errors = [] + for key in ("ELASTIC_USER", "ELASTIC_PASSWORD", "KIBANA_USER", "KIBANA_PASSWORD"): + if not values.get(key): + errors.append(f"{key} must be set in the selected users_elasticsearch.env") + salt = values.get("OPENSEARCH_COMPLIANCE_SALT", "") + if len(salt) < 16 or not salt.isascii(): + errors.append("OPENSEARCH_COMPLIANCE_SALT must contain at least 16 ASCII characters") + master_key = values.get("OPENSEARCH_DATASOURCE_MASTER_KEY", "") + if len(master_key) not in (16, 24, 32) or not master_key.isascii(): + errors.append("OPENSEARCH_DATASOURCE_MASTER_KEY must contain 16, 24, or 32 ASCII characters") + versions = [] + for key in ("ELASTICSEARCH_DOCKER_IMAGE", "ELASTICSEARCH_KIBANA_DOCKER_IMAGE"): + match = re.search(r":(\d+\.\d+\.\d+)(?:@sha256:[0-9a-f]{64})?$", values.get(key, "")) + if match: + versions.append(match[1]) + if len(versions) == 2 and versions[0] != versions[1]: + errors.append("OpenSearch and OpenSearch Dashboards image release versions must match") + return errors + + +def main() -> int: + errors = setting_errors(os.environ) + if errors: + print("Invalid OpenSearch deployment settings:", file=sys.stderr) + for error in errors: + print(f"- {error}", file=sys.stderr) + return 1 + print("Search credentials, encryption settings, and image versions are valid.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/tests/smoke_elastic_services.sh b/scripts/tests/smoke_elastic_services.sh index a2c82d47f..7de8c7326 100755 --- a/scripts/tests/smoke_elastic_services.sh +++ b/scripts/tests/smoke_elastic_services.sh @@ -65,7 +65,7 @@ for port in "${ELASTICSEARCH_PORT_LIST[@]}"; do done KIBANA_CHECKS=( - "kibana|https://${KIBANA_HOST}:${KIBANA_PORT}/" + "kibana|https://${KIBANA_HOST}:${KIBANA_PORT}/api/status" ) if [[ "$START_SERVICES" != "0" ]]; then @@ -74,8 +74,8 @@ if [[ "$START_SERVICES" != "0" ]]; then exit 1 fi - echo "Starting Elasticsearch services using 'make -C deploy start-elastic'." - make -C "${ROOT_DIR}/deploy" start-elastic + echo "Starting the search cluster and Dashboards." + make -C "${ROOT_DIR}/deploy" start-elastic-cluster start-kibana fi set_smoke_allowed_codes 200 @@ -85,13 +85,41 @@ wait_for_checks \ "$ELASTICSEARCH_DELAY_SECONDS" \ "${ELASTICSEARCH_CHECKS[@]}" +# Check the actual internal-user credentials as well as the admin certificate. +: "${ELASTIC_USER:?ELASTIC_USER is required for search smoke checks}" +: "${ELASTIC_PASSWORD:?ELASTIC_PASSWORD is required for search smoke checks}" +set_smoke_curl_args --user "${ELASTIC_USER}:${ELASTIC_PASSWORD}" +wait_for_checks \ + "Search password authentication" \ + "$ELASTICSEARCH_RETRIES" \ + "$ELASTICSEARCH_DELAY_SECONDS" \ + "${ELASTICSEARCH_CHECKS[@]}" + +if [[ "$SEARCH_BACKEND" == "opensearch" ]]; then + # Strip the digest before extracting the human-readable release tag. + image_tag="${ELASTICSEARCH_DOCKER_IMAGE%@*}" + expected_version="${image_tag##*:}" + if [[ "$expected_version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + actual_version="$(curl --silent --show-error --fail \ + --connect-timeout "$SMOKE_CONNECT_TIMEOUT" --max-time "$SMOKE_MAX_TIME" \ + --cacert "$SMOKE_CA_CERT" --user "${ELASTIC_USER}:${ELASTIC_PASSWORD}" \ + "https://${HOST}:${ELASTICSEARCH_PORT_LIST[0]}/" \ + | python3 -c 'import json,sys; print(json.load(sys.stdin)["version"]["number"])')" + if [[ "$actual_version" != "$expected_version" ]]; then + echo "FAIL: OpenSearch version ${actual_version} differs from selected ${expected_version}" >&2 + exit 1 + fi + echo "OK: OpenSearch version ${actual_version}" + fi +fi + if [[ "$KIBANA_GRACE_SECONDS" -gt 0 ]]; then echo "Elasticsearch is healthy. Waiting ${KIBANA_GRACE_SECONDS}s before Kibana checks." sleep "$KIBANA_GRACE_SECONDS" fi -set_smoke_curl_args -set_smoke_allowed_codes 200 301 302 303 307 308 +set_smoke_curl_args --user "${ELASTIC_USER}:${ELASTIC_PASSWORD}" +set_smoke_allowed_codes 200 wait_for_checks \ "Kibana" \ "$KIBANA_RETRIES" \ diff --git a/scripts/tests/test_opensearch_deployment.py b/scripts/tests/test_opensearch_deployment.py new file mode 100755 index 000000000..1871195f0 --- /dev/null +++ b/scripts/tests/test_opensearch_deployment.py @@ -0,0 +1,261 @@ +#!/usr/bin/env python3 +"""Check Compose/Helm parity and real certificate generation without a cluster.""" + +from __future__ import annotations + +import importlib.util +import json +import os +import re +import shutil +import subprocess +import tempfile +import unittest +from pathlib import Path +from typing import ClassVar + +from check_opensearch_settings import setting_errors + +ROOT = Path(__file__).resolve().parents[2] + + +def yaml_documents(text: str) -> list[dict]: + import yaml + + return [document for document in yaml.safe_load_all(text) if isinstance(document, dict)] + + +def reference_environment() -> dict[str, str]: + return {key: value for key, value in os.environ.items() if not key.startswith("COGSTACK_")} + + +@unittest.skipUnless( + shutil.which("helm") and shutil.which("docker") and importlib.util.find_spec("yaml"), + "Helm, Docker Compose, and PyYAML are required", +) +class OpenSearchDeploymentTest(unittest.TestCase): + compose: ClassVar[dict] + documents: ClassVar[list[dict]] + + @classmethod + def setUpClass(cls) -> None: + result = subprocess.run( + ["bash", "-c", 'source "$1" >/dev/null && docker compose --profile "*" -f "$2" config --format json', + "read-compose", str(ROOT / "deploy/export_env_vars.sh"), str(ROOT / "deploy/services.yml")], + env=reference_environment(), text=True, capture_output=True, check=True, timeout=30, + ) + cls.compose = json.loads(result.stdout)["services"] + result = cls.render() + if result.returncode: + raise RuntimeError(result.stderr) + cls.documents = yaml_documents(result.stdout) + + @staticmethod + def render(*args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["helm", "template", "cogstack-opensearch", str(ROOT / "deploy/charts/opensearch"), + "-f", str(ROOT / "deploy/helm/opensearch.values.yaml"), "--namespace", "cogstack", *args], + env=reference_environment(), text=True, capture_output=True, timeout=30, + ) + + def resource(self, kind: str) -> dict: + return next(document for document in self.documents if document["kind"] == kind) + + def test_default_versions_and_digests_match_compose(self) -> None: + stateful = self.resource("StatefulSet")["spec"]["template"]["spec"] + dashboards = self.resource("Deployment")["spec"]["template"]["spec"] + search_image = self.compose["elasticsearch-1"]["image"] + helm_search_image = stateful["containers"][0]["image"] + self.assertTrue(all(container["image"] == helm_search_image for container in stateful["initContainers"])) + metadata = yaml_documents((ROOT / "deploy/charts/opensearch/Chart.yaml").read_text())[0] + values = yaml_documents((ROOT / "deploy/charts/opensearch/values.yaml").read_text())[0] + for component, helm_image, compose_image in ( + ("opensearch", helm_search_image, search_image), + ("dashboards", dashboards["containers"][0]["image"], self.compose["kibana"]["image"]), + ): + image = values[component]["image"] + self.assertEqual(image["tag"], metadata["appVersion"]) + self.assertEqual(compose_image, f"{image['repository']}:{image['tag']}@{image['digest']}") + self.assertEqual(helm_image, f"{image['repository']}@{image['digest']}") + + def test_generated_secret_matches_compose_and_all_references_resolve(self) -> None: + secret = self.resource("Secret") + data = secret["stringData"] + search_env = self.compose["elasticsearch-1"]["environment"] + dashboards_env = self.compose["kibana"]["environment"] + for key in ("OPENSEARCH_INITIAL_ADMIN_PASSWORD", "OPENSEARCH_COMPLIANCE_SALT", + "OPENSEARCH_DATASOURCE_MASTER_KEY"): + self.assertTrue(data[key] == search_env[key], f"Shared setting differs: {key}") + for key in ("KIBANA_USER", "KIBANA_PASSWORD"): + self.assertTrue(data[key] == dashboards_env[key], f"Shared setting differs: {key}") + self.assertEqual(search_env["ELASTIC_USER"], "admin") + self.assertEqual(setting_errors(search_env), []) + for kind in ("StatefulSet", "Deployment"): + for container in self.resource(kind)["spec"]["template"]["spec"]["containers"]: + for entry in container.get("env", []): + ref = entry.get("valueFrom", {}).get("secretKeyRef") + if ref: + self.assertEqual(ref["name"], secret["metadata"]["name"]) + self.assertIn(ref["key"], data) + + def test_shared_security_and_runtime_configs_are_bundled_unchanged(self) -> None: + maps = [document["data"] for document in self.documents if document["kind"] == "ConfigMap"] + expected = { + "opensearch.yml": ROOT / "services/elasticsearch/config/opensearch.yml", + "opensearch_dashboards.yml": ROOT / "services/kibana/config/opensearch.yml", + "internal_users.yml": ROOT / "security/es_roles/opensearch/internal_users.yml", + "search-entrypoint.sh": ROOT / "services/elasticsearch/search-entrypoint.sh", + } + for key, path in expected.items(): + actual = next(data[key] for data in maps if key in data) + self.assertEqual(actual.strip(), path.read_text().strip(), key) + dashboards_config = next( + data["opensearch_dashboards.yml"] for data in maps if "opensearch_dashboards.yml" in data + ) + self.assertIn("opensearch.ssl.verificationMode: full", dashboards_config) + self.assertIn("opensearch.username: ${KIBANA_USER}\n", dashboards_config) + self.assertIn("opensearch.password: ${KIBANA_PASSWORD}\n", dashboards_config) + self.assertNotIn("opensearch.ssl.certificate:", dashboards_config) + self.assertNotIn("opensearch.ssl.key:", dashboards_config) + mounts = self.resource("Deployment")["spec"]["template"]["spec"]["containers"][0]["volumeMounts"] + self.assertFalse(any("admin." in mount.get("subPath", "") for mount in mounts)) + + def test_external_credentials_secret_mode(self) -> None: + result = self.render("--set", "credentials.create=false", "--set", "credentials.existingSecret=external-creds") + self.assertEqual(result.returncode, 0, result.stderr) + documents = yaml_documents(result.stdout) + self.assertFalse(any(document["kind"] == "Secret" for document in documents)) + for document in documents: + if document["kind"] in ("StatefulSet", "Deployment"): + for container in document["spec"]["template"]["spec"]["containers"]: + for entry in container.get("env", []): + ref = entry.get("valueFrom", {}).get("secretKeyRef") + if ref: + self.assertEqual(ref["name"], "external-creds") + + def test_selected_profile_credentials_reach_helm_and_compose(self) -> None: + with tempfile.TemporaryDirectory() as directory: + profile = Path(directory) + (profile / "security").mkdir() + contents = (ROOT / "security/env/users_elasticsearch.env").read_text() + contents = re.sub( + r"^KIBANA_PASSWORD=.*$", "KIBANA_PASSWORD=fixture-selected-password", contents, flags=re.M, + ) + (profile / "security/users_elasticsearch.env").write_text(contents) + environment = {**reference_environment(), "COGSTACK_CONFIG_DIR": str(profile)} + result = subprocess.run( + ["make", "-s", "-C", str(ROOT / "deploy"), "helm-template-opensearch"], + env=environment, text=True, capture_output=True, timeout=30, + ) + self.assertEqual(result.returncode, 0, result.stderr) + secret = next(document for document in yaml_documents(result.stdout) if document["kind"] == "Secret") + self.assertTrue(secret["stringData"]["KIBANA_PASSWORD"] == "fixture-selected-password") + result = subprocess.run( + ["bash", "-c", 'source "$1" >/dev/null && docker compose --profile "*" -f "$2" config --format json', + "read-compose", str(ROOT / "deploy/export_env_vars.sh"), str(ROOT / "deploy/services.yml")], + env=environment, text=True, capture_output=True, check=True, timeout=30, + ) + password = json.loads(result.stdout)["services"]["kibana"]["environment"]["KIBANA_PASSWORD"] + self.assertTrue(password == secret["stringData"]["KIBANA_PASSWORD"]) + + def test_invalid_credentials_and_incomplete_tls_mapping_fail_rendering(self) -> None: + cases = [ + (("--set", "credentials.create=false"), "credentials.existingSecret"), + (("--set", "opensearch.replicas=4"), "opensearchNodeFiles"), + (("--set", "certificates.opensearchSecretName="), "opensearchSecretName"), + ] + for args, message in cases: + with self.subTest(args=args): + result = self.render(*args) + self.assertNotEqual(result.returncode, 0) + self.assertIn(message, result.stderr) + for key in ("OPENSEARCH_COMPLIANCE_SALT", "OPENSEARCH_DATASOURCE_MASTER_KEY", "KIBANA_PASSWORD"): + with self.subTest(key=key), tempfile.TemporaryDirectory() as directory: + filename = Path(directory) / "invalid.env" + text = (ROOT / "security/env/users_elasticsearch.env").read_text() + filename.write_text(re.sub(rf"^{key}=.*$", f"{key}=", text, flags=re.MULTILINE)) + result = self.render("--set-file", f"usersEnvFile.raw={filename}") + self.assertNotEqual(result.returncode, 0) + self.assertIn(key, result.stderr) + + def test_shared_settings_reject_invalid_lengths_and_version_mismatches(self) -> None: + values = self.compose["elasticsearch-1"]["environment"] + for change in ( + {"OPENSEARCH_COMPLIANCE_SALT": ""}, {"OPENSEARCH_COMPLIANCE_SALT": "x" * 15}, + {"OPENSEARCH_DATASOURCE_MASTER_KEY": "x" * 20}, {"KIBANA_PASSWORD": ""}, + {"ELASTICSEARCH_KIBANA_DOCKER_IMAGE": "opensearchproject/opensearch-dashboards:3.5.0"}, + ): + with self.subTest(keys=list(change)): + self.assertTrue(setting_errors({**values, **change})) + + +@unittest.skipUnless(shutil.which("openssl") and shutil.which("keytool"), "OpenSSL and Java keytool are required") +class OpenSearchCertificateTest(unittest.TestCase): + temp: ClassVar[tempfile.TemporaryDirectory[str]] + workspace: ClassVar[Path] + env: ClassVar[dict[str, str]] + certificates: ClassVar[Path] + + @classmethod + def setUpClass(cls) -> None: + cls.temp = tempfile.TemporaryDirectory(prefix="cogstack-tls-") + cls.addClassCleanup(cls.temp.cleanup) + cls.workspace = Path(cls.temp.name) + cls.env = reference_environment() + for relative in ("security/scripts", "security/env", "security/templates", "deploy"): + (cls.workspace / relative).mkdir(parents=True) + for source in (ROOT / "security/scripts").glob("*.sh"): + shutil.copy2(source, cls.workspace / "security/scripts" / source.name) + for name in ("certificates_general.env", "certificates_elasticsearch.env", "users_elasticsearch.env"): + source = ROOT / "security/env" / name + text = source.read_text().replace("ROOT_CERTIFICATE_KEY_SIZE=4096", "ROOT_CERTIFICATE_KEY_SIZE=2048") + text = text.replace("ES_KEY_SIZE=4096", "ES_KEY_SIZE=2048") + (cls.workspace / "security/env" / name).write_text(text) + shutil.copy2(ROOT / "security/templates/ssl-extensions-x509.cnf", cls.workspace / "security/templates") + shutil.copy2(ROOT / "deploy/config_files.sh", cls.workspace / "deploy") + for script, args in ( + ("create_root_ca_cert.sh", []), + ("create_opensearch_node_cert.sh", ["elasticsearch-1", "elasticsearch-2", "elasticsearch-3"]), + ("create_opensearch_client_admin_certs.sh", []), + ): + cls.command("bash", script, *args, cwd=cls.workspace / "security/scripts") + cls.certificates = cls.workspace / "security/certificates/elastic/opensearch" + + @classmethod + def command(cls, *args: str, cwd: Path | None = None) -> str: + result = subprocess.run( + list(args), cwd=cwd, env=cls.env, text=True, capture_output=True, timeout=90, + ) + if result.returncode: + raise RuntimeError(f"Certificate command failed: {args[0]}\n{result.stderr}") + return result.stdout + + def test_node_certificates_verify_for_compose_and_helm_service_names(self) -> None: + ca = self.certificates / "elastic-stack-ca.crt.pem" + for ordinal in (1, 2, 3): + name = f"elasticsearch-{ordinal}" + cert = self.certificates / "elasticsearch" / name / f"{name}.crt" + for hostname in (name, "localhost", "cogstack-opensearch-client.cogstack.svc", + "cogstack-opensearch-client.cogstack.svc.cluster.local"): + with self.subTest(node=name, hostname=hostname): + self.command( + "openssl", "verify", "-CAfile", str(ca), "-purpose", "sslserver", + "-verify_hostname", hostname, str(cert), + ) + self.command("openssl", "verify", "-CAfile", str(ca), "-verify_ip", "127.0.0.1", str(cert)) + key = cert.with_suffix(".key") + self.command("openssl", "pkey", "-in", str(key), "-check", "-noout") + + def test_dashboards_certificate_verifies_for_localhost_and_kubernetes(self) -> None: + cert = self.certificates / "es_kibana_client.pem" + ca = self.certificates / "elastic-stack-ca.crt.pem" + for hostname in ("localhost", "kibana", "cogstack-opensearch-dashboards.cogstack.svc"): + with self.subTest(hostname=hostname): + self.command( + "openssl", "verify", "-CAfile", str(ca), "-purpose", "sslserver", + "-verify_hostname", hostname, str(cert), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/security/env/certificates_elasticsearch.env b/security/env/certificates_elasticsearch.env index a46ea7332..c6f8137a6 100644 --- a/security/env/certificates_elasticsearch.env +++ b/security/env/certificates_elasticsearch.env @@ -33,7 +33,7 @@ ES_INSTANCE_ALTERNATIVE_3_NAME=elasticsearch-3-dev ES_CLIENT_CERT_NAME="es_kibana_client" ES_CLIENT_SUBJ_LINE="/C=UK/ST=London/L=UK/O=cogstack/OU=cogstack/CN=client" -ES_CLIENT_SUBJ_ALT_NAMES="subjectAltName=DNS:kibana,DNS:metricbeat-1,DNS:metricbeat-2,DNS:metricbeat-3,DNS:elasticsearch-1,DNS:elasticsearch-2,DNS:elasticsearch-3,DNS:nifi,DNS:cogstack,DNS:*.cogstack" +ES_CLIENT_SUBJ_ALT_NAMES="subjectAltName=DNS:kibana,DNS:metricbeat-1,DNS:metricbeat-2,DNS:metricbeat-3,DNS:elasticsearch-1,DNS:elasticsearch-2,DNS:elasticsearch-3,DNS:nifi,DNS:cogstack,DNS:*.cogstack,DNS:localhost,IP:127.0.0.1,DNS:cogstack-kibana,DNS:cogstack-opensearch-dashboards.cogstack.svc,DNS:cogstack-opensearch-dashboards.cogstack.svc.cluster.local" ES_ADMIN_SUBJ_LINE="/C=UK/ST=London/L=UK/O=cogstack/OU=cogstack/CN=admin" @@ -42,4 +42,4 @@ ES_ADMIN_SUBJ_LINE="/C=UK/ST=London/L=UK/O=cogstack/OU=cogstack/CN=admin" ES_NODE_SUBJ_LINE="/C=UK/ST=London/L=UK/O=cogstack/OU=cogstack" # subjectAltName=$cert_name gets appended within the script, example ES_NODE_SUBJ_ALT_NAMES="subjectAltName=DNS:$cert_name,"${ES_NODE_SUBJ_ALT_NAMES} -ES_NODE_SUBJ_ALT_NAMES="DNS:kibana,DNS:metricbeat-1,DNS:metricbeat-2,DNS:metricbeat-3,DNS:elasticsearch-1,DNS:elasticsearch-2,DNS:elasticsearch-3,DNS:nifi,DNS:cogstack,DNS:*.cogstack" \ No newline at end of file +ES_NODE_SUBJ_ALT_NAMES="DNS:kibana,DNS:metricbeat-1,DNS:metricbeat-2,DNS:metricbeat-3,DNS:elasticsearch-1,DNS:elasticsearch-2,DNS:elasticsearch-3,DNS:nifi,DNS:cogstack,DNS:*.cogstack,DNS:localhost,IP:127.0.0.1,DNS:cogstack-elasticsearch-1,DNS:cogstack-elasticsearch-2,DNS:cogstack-elasticsearch-3,DNS:cogstack-opensearch-client.cogstack.svc,DNS:cogstack-opensearch-client.cogstack.svc.cluster.local" diff --git a/security/env/users_elasticsearch.env b/security/env/users_elasticsearch.env index 387b1d298..55a8c1df2 100644 --- a/security/env/users_elasticsearch.env +++ b/security/env/users_elasticsearch.env @@ -2,18 +2,20 @@ # admin account, ELASTIC_USER must always be set to "elastic" if you are using NATIVE ELASTICSEARCH, # for opensearch it is "admin" -ELASTIC_USER=elastic +ELASTIC_USER=admin # admin account password ELASTIC_PASSWORD=admin +# Read only by the demo installer, which this deployment disables. +# Actual credentials come from internal_users.yml; ELASTIC_PASSWORD must match its admin hash. OPENSEARCH_INITIAL_ADMIN_PASSWORD=Kibanaserver!1 # Public development values, identical on all OpenSearch nodes. # Replace through deployment-local configuration or an external secret store. # Keep the salt stable (16 ASCII characters) and preserve the datasource key # (16, 24, or 32 ASCII characters) to decrypt existing datasource credentials. -OPENSEARCH_COMPLIANCE_SALT= -OPENSEARCH_DATASOURCE_MASTER_KEY= +OPENSEARCH_COMPLIANCE_SALT=0123456789abcdef +OPENSEARCH_DATASOURCE_MASTER_KEY=0123456789abcdef0123456789abcdef # specify here the kibana user and password that # will be used by kibana when connecting to ES diff --git a/security/scripts/create_opensearch_client_admin_certs.sh b/security/scripts/create_opensearch_client_admin_certs.sh index 6c10cde68..5c83cee96 100755 --- a/security/scripts/create_opensearch_client_admin_certs.sh +++ b/security/scripts/create_opensearch_client_admin_certs.sh @@ -22,7 +22,6 @@ set -euo pipefail source "$(dirname "${BASH_SOURCE[0]}")/load_config.sh" || exit 1 -SECURITY_TEMPLATES_FOLDER="../templates/" SECURITY_CERTIFICATES_FOLDER="../certificates/" ROOT_CERTIFICATES_FOLDER="${SECURITY_CERTIFICATES_FOLDER}root/" @@ -38,6 +37,7 @@ source "$(cogstack_env_file security/env/certificates_elasticsearch.env)" || exi : "${ROOT_CERTIFICATE_NAME:?Must be set in certificates_general.env}" : "${ROOT_CERTIFICATE_KEYSTORE_PASSWORD:?Must be set in certificates_general.env}" : "${ES_CLIENT_SUBJ_LINE:?Must be set in certificates_elasticsearch.env}" +: "${ES_CLIENT_SUBJ_ALT_NAMES:?Must be set in certificates_elasticsearch.env}" : "${ES_CLIENT_CERT_NAME:?Must be set in certificates_elasticsearch.env}" : "${ES_ADMIN_SUBJ_LINE:?Must be set in certificates_elasticsearch.env}" : "${ES_KEY_SIZE:?Must be set in certificates_elasticsearch.env}" @@ -54,7 +54,6 @@ echo "========================================================================== CA_ROOT_CERT="${ROOT_CERTIFICATES_FOLDER}${ROOT_CERTIFICATE_NAME}.pem" CA_ROOT_KEY="${ROOT_CERTIFICATES_FOLDER}${ROOT_CERTIFICATE_NAME}.key" -EXT_FILE="${SECURITY_TEMPLATES_FOLDER}ssl-extensions-x509.cnf" # === Client cert === echo "Generating a key for: $ES_CLIENT_CERT_NAME" @@ -71,7 +70,13 @@ openssl x509 -req -days "$ES_CERTIFICATE_TIME_VALIDITY_IN_DAYS" \ -in "$ES_CLIENT_CERT_NAME.csr" \ -CA "$CA_ROOT_CERT" -CAkey "$CA_ROOT_KEY" -CAcreateserial \ -out "$ES_CLIENT_CERT_NAME.pem" \ - -extensions v3_leaf -extfile "$EXT_FILE" + -extfile <(printf '%s\n' \ + 'basicConstraints=critical,CA:FALSE' \ + 'keyUsage=critical,digitalSignature,keyEncipherment' \ + 'extendedKeyUsage=serverAuth,clientAuth' \ + 'subjectKeyIdentifier=hash' \ + 'authorityKeyIdentifier=keyid,issuer' \ + "$ES_CLIENT_SUBJ_ALT_NAMES") mv "$ES_CLIENT_CERT_NAME"* "$OPENSEARCH_ES_CERTIFICATES_FOLDER" diff --git a/security/scripts/create_opensearch_node_cert.sh b/security/scripts/create_opensearch_node_cert.sh index 33fcd5cf0..94679598b 100755 --- a/security/scripts/create_opensearch_node_cert.sh +++ b/security/scripts/create_opensearch_node_cert.sh @@ -29,7 +29,6 @@ set -euo pipefail source "$(dirname "${BASH_SOURCE[0]}")/load_config.sh" || exit 1 -SECURITY_TEMPLATES_FOLDER="../templates/" SECURITY_CERTIFICATES_FOLDER="../certificates/" ROOT_CERTIFICATES_FOLDER="${SECURITY_CERTIFICATES_FOLDER}root/" @@ -119,8 +118,13 @@ generate_node_cert() { -CAkey "$CA_ROOT_KEY" \ -CAcreateserial \ -out "${cert_name}.crt" \ - -extensions v3_leaf \ - -extfile "${SECURITY_TEMPLATES_FOLDER}ssl-extensions-x509.cnf" + -extfile <(printf '%s\n' \ + 'basicConstraints=critical,CA:FALSE' \ + 'keyUsage=critical,digitalSignature,keyEncipherment' \ + 'extendedKeyUsage=serverAuth,clientAuth' \ + 'subjectKeyIdentifier=hash' \ + 'authorityKeyIdentifier=keyid,issuer' \ + "$subject_alt_name") # === Create Java Keystore === echo "🔐 Creating Java Keystore (.jks)..." diff --git a/services/kibana/config/opensearch.yml b/services/kibana/config/opensearch.yml index 61c35bb39..641135232 100644 --- a/services/kibana/config/opensearch.yml +++ b/services/kibana/config/opensearch.yml @@ -57,8 +57,8 @@ opensearchDashboards.index: ".opensearch_dashboards" # the username and password that the OpenSearch Dashboards server uses to perform maintenance on the OpenSearch Dashboards # index at startup. Your OpenSearch Dashboards users still need to authenticate with OpenSearch, which # is proxied through the OpenSearch Dashboards server. -opensearch.username: ${KIBANA_USER:-"kibanaserver"} -opensearch.password: ${KIBANA_PASSWORD:-"kibanaserver"} +opensearch.username: ${KIBANA_USER} +opensearch.password: ${KIBANA_PASSWORD} # Enables SSL and paths to the PEM-format SSL certificate and SSL key files, respectively. # These settings enable SSL for outgoing requests from the OpenSearch Dashboards server to the browser. @@ -66,11 +66,8 @@ server.ssl.enabled: true server.ssl.certificate: "/certificates/elastic/opensearch/es_kibana_client.pem" server.ssl.key: "/certificates/elastic/opensearch/es_kibana_client.key" -# Optional settings that provide the paths to the PEM-format SSL certificate and key files. -# These files are used to verify the identity of OpenSearch Dashboards to OpenSearch and are required when -# xpack.security.http.ssl.client_authentication in OpenSearch is set to required. -opensearch.ssl.certificate: "/certificates/elastic/opensearch/admin.crt" -opensearch.ssl.key: "/certificates/elastic/opensearch/admin.key.pem" +# Outgoing requests use KIBANA_USER/KIBANA_PASSWORD and verify the server CA. +# Do not send an admin client certificate: that would bypass the service account. # Optional setting that enables you to specify a path to the PEM file for the certificate # authority for your OpenSearch instance. From 7c8a9315b44c78d5427c04947354604dbb032e2e Mon Sep 17 00:00:00 2001 From: Vlad Dinu <62345326+vladd-bit@users.noreply.github.com> Date: Thu, 29 Oct 2026 14:16:21 +0000 Subject: [PATCH 14/28] Potential fix for pull request finding 'Imprecise assert' Co-authored-by: Copilot Autofix powered by AI <223894421+github-code-quality[bot]@users.noreply.github.com> --- scripts/tests/test_opensearch_deployment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/tests/test_opensearch_deployment.py b/scripts/tests/test_opensearch_deployment.py index 1871195f0..3e48a2385 100755 --- a/scripts/tests/test_opensearch_deployment.py +++ b/scripts/tests/test_opensearch_deployment.py @@ -156,7 +156,7 @@ def test_selected_profile_credentials_reach_helm_and_compose(self) -> None: env=environment, text=True, capture_output=True, check=True, timeout=30, ) password = json.loads(result.stdout)["services"]["kibana"]["environment"]["KIBANA_PASSWORD"] - self.assertTrue(password == secret["stringData"]["KIBANA_PASSWORD"]) + self.assertEqual(password, secret["stringData"]["KIBANA_PASSWORD"]) def test_invalid_credentials_and_incomplete_tls_mapping_fail_rendering(self) -> None: cases = [ From 651a5acfb171072a837b212bb2f655fe050c5885 Mon Sep 17 00:00:00 2001 From: Vlad Dinu <62345326+vladd-bit@users.noreply.github.com> Date: Thu, 29 Oct 2026 14:16:30 +0000 Subject: [PATCH 15/28] Potential fix for pull request finding 'Imprecise assert' Co-authored-by: Copilot Autofix powered by AI <223894421+github-code-quality[bot]@users.noreply.github.com> --- scripts/tests/test_opensearch_deployment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/tests/test_opensearch_deployment.py b/scripts/tests/test_opensearch_deployment.py index 3e48a2385..5487572b0 100755 --- a/scripts/tests/test_opensearch_deployment.py +++ b/scripts/tests/test_opensearch_deployment.py @@ -149,7 +149,7 @@ def test_selected_profile_credentials_reach_helm_and_compose(self) -> None: ) self.assertEqual(result.returncode, 0, result.stderr) secret = next(document for document in yaml_documents(result.stdout) if document["kind"] == "Secret") - self.assertTrue(secret["stringData"]["KIBANA_PASSWORD"] == "fixture-selected-password") + self.assertEqual(secret["stringData"]["KIBANA_PASSWORD"], "fixture-selected-password") result = subprocess.run( ["bash", "-c", 'source "$1" >/dev/null && docker compose --profile "*" -f "$2" config --format json', "read-compose", str(ROOT / "deploy/export_env_vars.sh"), str(ROOT / "deploy/services.yml")], From 1876d40be002d6a0ff5a58143661fa68d6c520c7 Mon Sep 17 00:00:00 2001 From: Vlad Dinu <62345326+vladd-bit@users.noreply.github.com> Date: Thu, 29 Oct 2026 14:32:38 +0000 Subject: [PATCH 16/28] Potential fix for pull request finding 'Imprecise assert' Co-authored-by: Copilot Autofix powered by AI <223894421+github-code-quality[bot]@users.noreply.github.com> --- scripts/tests/test_configuration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/tests/test_configuration.py b/scripts/tests/test_configuration.py index 3a0ffa42e..61b76d11d 100755 --- a/scripts/tests/test_configuration.py +++ b/scripts/tests/test_configuration.py @@ -76,7 +76,7 @@ def test_compose_uses_profile_for_resources_ports_and_container_environment(self self.assertEqual(nifi["environment"]["NIFI_JVM_HEAP_MAX"], "1536m") # Compose escapes dollar signs when serializing a reusable model. password = nifi["environment"]["NIFI_SINGLE_USER_CREDENTIALS_PASSWORD"].replace("$$", "$") - self.assertTrue(password == selected_password) + self.assertEqual(password, selected_password) self.assertEqual(int(nifi["deploy"]["resources"]["limits"]["memory"]), 3 * 1024**3) self.assertTrue(any(port.get("published") == "18082" for port in nifi["ports"])) data_mount = next(volume for volume in nifi["volumes"] if volume["target"] == "/data") From fb73c4c73985b61526e493fbef6dbbff8e4b6cd6 Mon Sep 17 00:00:00 2001 From: Vlad Dinu <62345326+vladd-bit@users.noreply.github.com> Date: Thu, 29 Oct 2026 14:32:50 +0000 Subject: [PATCH 17/28] Potential fix for pull request finding 'Implicit string concatenation in a list' Co-authored-by: Copilot Autofix powered by AI <223894421+github-code-quality[bot]@users.noreply.github.com> --- scripts/tests/check_deployment_images.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/tests/check_deployment_images.py b/scripts/tests/check_deployment_images.py index 62ca1661d..d95067da7 100755 --- a/scripts/tests/check_deployment_images.py +++ b/scripts/tests/check_deployment_images.py @@ -57,7 +57,7 @@ def render_compose(path: Path) -> dict: [ "bash", "-c", 'source "$1" >/dev/null; set -euo pipefail; ' - 'exec docker compose --profile "*" -f "$2" config --format json', + + 'exec docker compose --profile "*" -f "$2" config --format json', "check-deployment-images", str(ROOT / "deploy/export_env_vars.sh"), str(path), ], text=True, capture_output=True, check=True, timeout=60, From 0a121332b19bb14428fc9c7f31cae26f5ff287a9 Mon Sep 17 00:00:00 2001 From: vladd-bit Date: Thu, 29 Oct 2026 14:56:35 +0000 Subject: [PATCH 18/28] MyPy fixes. --- nifi/user_python_extensions/record_decompress_cerner_blob.py | 2 +- typings/nifiapi/flowfiletransform.pyi | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nifi/user_python_extensions/record_decompress_cerner_blob.py b/nifi/user_python_extensions/record_decompress_cerner_blob.py index 952df054e..8b095c09a 100644 --- a/nifi/user_python_extensions/record_decompress_cerner_blob.py +++ b/nifi/user_python_extensions/record_decompress_cerner_blob.py @@ -320,7 +320,7 @@ def process(self, context: ProcessContext, flowFile: JavaObject) -> FlowFileTran for k in order_of_blobs_keys: v = concatenated_blob_sequence_order[k] - temporary_blob: bytes = b"" + temporary_blob: bytes | bytearray = b"" if self.binary_field_source_encoding == "base64": if not isinstance(v, str): diff --git a/typings/nifiapi/flowfiletransform.pyi b/typings/nifiapi/flowfiletransform.pyi index 65306797a..f9f369e76 100644 --- a/typings/nifiapi/flowfiletransform.pyi +++ b/typings/nifiapi/flowfiletransform.pyi @@ -15,5 +15,5 @@ class FlowFileTransformResult: self, relationship: str, attributes: dict[str, str], - contents: bytes | None = None, + contents: str | bytes | bytearray | None = None, ) -> None: ... From 6b35e490888b83c196d38db296ab38c99f03f630 Mon Sep 17 00:00:00 2001 From: Vlad Dinu <62345326+vladd-bit@users.noreply.github.com> Date: Thu, 29 Oct 2026 15:03:03 +0000 Subject: [PATCH 19/28] Potential fix for pull request finding 'CodeQL / Clear-text storage of sensitive information' Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- scripts/tests/test_configuration.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/tests/test_configuration.py b/scripts/tests/test_configuration.py index 61b76d11d..653057f54 100755 --- a/scripts/tests/test_configuration.py +++ b/scripts/tests/test_configuration.py @@ -63,8 +63,8 @@ def test_compose_uses_profile_for_resources_ports_and_container_environment(self "NIFI_JVM_HEAP_MAX": "1536m", "NIFI_DOCKER_RAM": "3g", "NIFI_OUTPUT_PORT": "18082", "NIFI_DATA_PATH": str(self.directory / "data with spaces"), }) - selected_password = "fixture-$value with spaces" - self.settings("users_nifi.env", {"NIFI_SINGLE_USER_CREDENTIALS_PASSWORD": selected_password}) + selected_value = "fixture-$value with spaces" + self.settings("users_nifi.env", {"NIFI_TEST_VALUE": selected_value}) for compose_file in ("services.yml", "services.dev.yml"): with self.subTest(compose_file=compose_file): result = self.run_bash( @@ -75,8 +75,8 @@ def test_compose_uses_profile_for_resources_ports_and_container_environment(self nifi = json.loads(result.stdout)["services"]["nifi"] self.assertEqual(nifi["environment"]["NIFI_JVM_HEAP_MAX"], "1536m") # Compose escapes dollar signs when serializing a reusable model. - password = nifi["environment"]["NIFI_SINGLE_USER_CREDENTIALS_PASSWORD"].replace("$$", "$") - self.assertEqual(password, selected_password) + value = nifi["environment"]["NIFI_TEST_VALUE"].replace("$$", "$") + self.assertEqual(value, selected_value) self.assertEqual(int(nifi["deploy"]["resources"]["limits"]["memory"]), 3 * 1024**3) self.assertTrue(any(port.get("published") == "18082" for port in nifi["ports"])) data_mount = next(volume for volume in nifi["volumes"] if volume["target"] == "/data") From 07b6c30232cd0a2ec376c689098580fde612aaf9 Mon Sep 17 00:00:00 2001 From: vladd-bit Date: Thu, 29 Oct 2026 17:42:42 +0000 Subject: [PATCH 20/28] Cleaned .gitignore and corrected stale image-tag(s). --- .../workflows/docker_elasticsearch_stack.yml | 9 ++ .github/workflows/docker_nifi.yml | 11 +++ .github/workflows/docker_nifi_nginx.yml | 8 ++ .github/workflows/smoke_elastic_services.yml | 22 ++++- .github/workflows/smoke_gitea_service.yml | 19 +++- .github/workflows/smoke_nifi_services.yml | 24 +++++- .github/workflows/smoke_postgres_services.yml | 17 +++- .gitignore | 86 ++++++++++--------- docs/deploy/reproducible.md | 8 ++ docs/nifi/main.md | 31 +++++-- security/env/README.md | 14 +-- 11 files changed, 191 insertions(+), 58 deletions(-) diff --git a/.github/workflows/docker_elasticsearch_stack.yml b/.github/workflows/docker_elasticsearch_stack.yml index 3a11459ab..c90be9b4a 100644 --- a/.github/workflows/docker_elasticsearch_stack.yml +++ b/.github/workflows/docker_elasticsearch_stack.yml @@ -4,7 +4,16 @@ on: push: branches: ['**'] tags: ['v*.*.*'] + paths: &paths + - ".github/workflows/docker_elasticsearch_stack.yml" + - ".github/workflows/_docker_template.yml" + - ".gitattributes" + - "services/elasticsearch/**" + - "services/kibana/**" + - "!**/*.md" + - "!**/README*" pull_request: + paths: *paths release: types: [published] workflow_dispatch: diff --git a/.github/workflows/docker_nifi.yml b/.github/workflows/docker_nifi.yml index 9c02b09d6..a1805b33c 100644 --- a/.github/workflows/docker_nifi.yml +++ b/.github/workflows/docker_nifi.yml @@ -4,7 +4,18 @@ on: push: branches: ['**'] tags: ['v*.*.*'] + paths: &paths + - ".github/workflows/docker_nifi.yml" + - ".github/workflows/_docker_template.yml" + - ".gitattributes" + - ".dockerignore" + - "nifi/**" + - "pyproject.toml" + - "uv.lock" + - "!**/*.md" + - "!**/README*" pull_request: + paths: *paths release: types: [published] workflow_dispatch: diff --git a/.github/workflows/docker_nifi_nginx.yml b/.github/workflows/docker_nifi_nginx.yml index 8e6a03a10..a8a80b4ff 100644 --- a/.github/workflows/docker_nifi_nginx.yml +++ b/.github/workflows/docker_nifi_nginx.yml @@ -4,7 +4,15 @@ on: push: branches: ['**'] tags: ['v*.*.*'] + paths: &paths + - ".github/workflows/docker_nifi_nginx.yml" + - ".github/workflows/_docker_template.yml" + - ".gitattributes" + - "services/nginx/**" + - "!**/*.md" + - "!**/README*" pull_request: + paths: *paths release: types: [published] workflow_dispatch: diff --git a/.github/workflows/smoke_elastic_services.yml b/.github/workflows/smoke_elastic_services.yml index 59bbd8367..1ed471045 100644 --- a/.github/workflows/smoke_elastic_services.yml +++ b/.github/workflows/smoke_elastic_services.yml @@ -1,9 +1,29 @@ name: smoke-elastic-services on: - pull_request: push: branches: ['**'] + paths: &paths + - ".github/workflows/smoke_elastic_services.yml" + - ".gitattributes" + - "deploy/Makefile" + - "deploy/export_env_vars.sh" + - "deploy/config_files.sh" + - "deploy/*.env" + - "security/env/*.env" + - "security/scripts/**" + - "security/templates/**" + - "deploy/services.yml" + - "services/elasticsearch/**" + - "services/kibana/**" + - "security/es_roles/**" + - "scripts/tests/check_opensearch_settings.py" + - "scripts/tests/smoke_elastic_services.sh" + - "scripts/tests/smoke_http_checks.sh" + - "!**/*.md" + - "!**/README*" + pull_request: + paths: *paths workflow_dispatch: concurrency: diff --git a/.github/workflows/smoke_gitea_service.yml b/.github/workflows/smoke_gitea_service.yml index 157277cca..c3ca88dcb 100644 --- a/.github/workflows/smoke_gitea_service.yml +++ b/.github/workflows/smoke_gitea_service.yml @@ -1,9 +1,26 @@ name: smoke-gitea-service on: - pull_request: push: branches: ['**'] + paths: &paths + - ".github/workflows/smoke_gitea_service.yml" + - ".gitattributes" + - "deploy/Makefile" + - "deploy/export_env_vars.sh" + - "deploy/config_files.sh" + - "deploy/*.env" + - "security/env/*.env" + - "security/scripts/**" + - "security/templates/**" + - "deploy/services.yml" + - "services/gitea/**" + - "scripts/tests/smoke_gitea_service.sh" + - "scripts/tests/smoke_http_checks.sh" + - "!**/*.md" + - "!**/README*" + pull_request: + paths: *paths workflow_dispatch: concurrency: diff --git a/.github/workflows/smoke_nifi_services.yml b/.github/workflows/smoke_nifi_services.yml index 63386c5c6..c9ff74cdc 100644 --- a/.github/workflows/smoke_nifi_services.yml +++ b/.github/workflows/smoke_nifi_services.yml @@ -1,9 +1,31 @@ name: smoke-nifi-services on: - pull_request: push: branches: ['**'] + paths: &paths + - ".github/workflows/smoke_nifi_services.yml" + - ".gitattributes" + - "deploy/Makefile" + - "deploy/export_env_vars.sh" + - "deploy/config_files.sh" + - "deploy/*.env" + - "security/env/*.env" + - "security/scripts/**" + - "security/templates/**" + - ".dockerignore" + - "nifi/**" + - "pyproject.toml" + - "uv.lock" + - "services/nginx/**" + - "deploy/services.dev.yml" + - "scripts/tests/check_nifi_resources.py" + - "scripts/tests/smoke_nifi_services.sh" + - "scripts/tests/smoke_http_checks.sh" + - "!**/*.md" + - "!**/README*" + pull_request: + paths: *paths workflow_dispatch: concurrency: diff --git a/.github/workflows/smoke_postgres_services.yml b/.github/workflows/smoke_postgres_services.yml index 54bdbad7d..67bc6c05f 100644 --- a/.github/workflows/smoke_postgres_services.yml +++ b/.github/workflows/smoke_postgres_services.yml @@ -1,9 +1,24 @@ name: smoke-postgres-services on: - pull_request: push: branches: ['**'] + paths: &paths + - ".github/workflows/smoke_postgres_services.yml" + - ".gitattributes" + - "deploy/Makefile" + - "deploy/export_env_vars.sh" + - "deploy/config_files.sh" + - "deploy/*.env" + - "security/env/*.env" + - "deploy/services.yml" + - "services/pgsamples" + - "services/pgsamples/**" + - "scripts/tests/smoke_postgres_services.sh" + - "!**/*.md" + - "!**/README*" + pull_request: + paths: *paths workflow_dispatch: concurrency: diff --git a/.gitignore b/.gitignore index 9b73edfe2..d7fa470f0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,24 +1,19 @@ -# IDE envs and system folders +# Editor files, Python environments, and build caches .DS_Store -**/.DS_Store -.idea -.vscode -.venv -.ruff_cache -.mypy_cache -venv -**__pycache__ -**/venv -*.pyc +.idea/ +.vscode/ +.venv/ +venv/ +__pycache__/ +*.py[cod] .pyc -build - -# Ignore setuptools metadata -*.egg-info/ +.ruff_cache/ +.mypy_cache/ +.pytest_cache/ +build/ *.egg-info -**/*.egg-info/ -# keys and certificates +# Generated keys, certificates, and certificate-tool downloads *.pem *.key *.jks @@ -26,33 +21,44 @@ build *.srl *.crt *.p12 - *elasticsearch-ssl-http.zip *es_native_certs_bundle_pem.zip *es_native_certs_bundle.zip -# env files with passwords or local paths -*.env -deploy/local/ -security/certificates/ -security/certificates/** -security/env/ -security/env/** -security/scripts/nifi_toolkit/ -security/scripts/nifi_toolkit/** -security/templates/ -security/templates/** - -# docs -docs/build/* -site/ +/security/certificates/* +!/security/certificates/README.md +/security/scripts/nifi_toolkit/ -# Ignore all .env files at any level -.env +# Local configuration stays ignored; public templates can be committed. *.env -**/*.env +.env.* !*.env.template -!**/*.env.template -!deploy/ollama.env -!deploy/sglang.env -!deploy/openwebui.env +/deploy/local/ +/security/env/* +!/security/env/README.md +!/security/env/*.env.template + +# Checked-in deployment defaults contain public development settings. +!/deploy/database.env +!/deploy/elasticsearch.env +!/deploy/general.env +!/deploy/gitea.env +!/deploy/network_settings.env +!/deploy/nginx.env +!/deploy/nifi.env +!/deploy/ollama.env +!/deploy/openwebui.env +!/deploy/project.env +!/deploy/sglang.env +!/deploy/telemetry.env +!/security/env/certificates_elasticsearch.env +!/security/env/certificates_general.env +!/security/env/certificates_nifi.env +!/security/env/users_database.env +!/security/env/users_elasticsearch.env +!/security/env/users_nginx.env +!/security/env/users_nifi.env + +# Documentation output +/docs/build/ +/site/ diff --git a/docs/deploy/reproducible.md b/docs/deploy/reproducible.md index 23e253817..d1e1f2c3b 100644 --- a/docs/deploy/reproducible.md +++ b/docs/deploy/reproducible.md @@ -27,6 +27,14 @@ CI checks these recorded commits with read-only repository permissions. It no longer advances dependencies or pushes synchronization commits when a branch changes. +Image builds and live service smoke tests run for changes to their source, +deployment configuration, helper scripts, or workflow files. Markdown and +README changes skip these expensive workflows; documentation checks still run. +Build workflows also run for version tag pushes, published releases, and manual +dispatches. Smoke workflows can be run manually when a full service check is +needed. Push and pull-request triggers share the same path filters. See +[GitHub's path-filter behavior](https://docs.github.com/en/actions/reference/workflows-and-actions/workflow-syntax#onpushpull_requestpull_request_targetpathspaths-ignore). + ## Upgrade submodules explicitly In a development checkout, run: diff --git a/docs/nifi/main.md b/docs/nifi/main.md index 52cd70999..27e8873eb 100644 --- a/docs/nifi/main.md +++ b/docs/nifi/main.md @@ -24,9 +24,7 @@ Avro Schema:[official documentation](https://avro.apache.org/docs/1.11.1/) ```text ├── Dockerfile - contains the base definition of the NiFi image along with all the packages/addons installed ├── conf - NiFi configuration files, this folder is mounted on the NiFi service container at runtime, it needs to have read & write permissions by the user -├── devel - custom folder that is mounted on the NiFi container where you may place your own scripts, again, read & write permissions required -├── drivers - drivers used for DB connections, currently PostgreSQL and MSSQL -├── nifi-app.log - log file mounted directly from the container for easy log checking +├── drivers - JDBC drivers used for database connections ├── user_schemas - Avro schemas used within workflows, it can also contain other schemas used in specific custom processors ├── user_scripts - custom scripts used in workflows, you can put them here ├── user_python_extensions - Python FlowFileTransform processors exposed to NiFi's extension framework @@ -37,6 +35,15 @@ For user script organization and usage guidelines, see [user scripts](user_scrip For Python extension processors, see [Python extensions](user_python_extensions.md). For end-to-end scripting patterns (including `ExecuteStreamCommand` stdin/stdout and args), see [processor scripting guide](processor_scripting.md). +Runtime logs are stored in the `nifi-vol-logs` volume at +`/opt/nifi/nifi-current/logs` inside the container. View startup and console logs +from the repository root with: + +```bash +source deploy/export_env_vars.sh +docker compose -f deploy/services.yml logs --tail=100 -f nifi +``` + ## Custom Docker image For any deployment it is recommended to build and use the custom Docker image of Apache NiFi that will contain all the necessary configuration, drivers, custom user scripts and workflows. @@ -51,7 +58,8 @@ See [Repeatable deployments](../deploy/reproducible.md) for the update procedure ## Apache NiFi configuration -The main configuration files for NiFi are provided in [`conf`](https://github.com/CogStack/CogStack-Nifi/conf) directory. +The main configuration files for NiFi are provided in the +[`nifi/conf`](https://github.com/CogStack/CogStack-NiFi/tree/main/nifi/conf) directory. This section provides only a brief description of the most useful properties that you may need to modify to fit your own setup.
@@ -158,7 +166,9 @@ To build the NiFi image without starting services, run this command from the rep make -C deploy build-nifi-image ``` -The Make target loads the deployment environment and tags the image using `NIFI_DOCKER_IMAGE` from `deploy/nifi.env`. +The Make target loads the selected deployment environment and tags the image +using `NIFI_LOCAL_DOCKER_IMAGE` (default: `cogstack/cogstack-nifi:local`). +`NIFI_DOCKER_IMAGE` selects the published image used by `services.yml`. ### `{bootstrap.conf}` @@ -166,11 +176,14 @@ The Make target loads the deployment environment and tags the image using `NIFI_ This file allows users to configure settings for how NiFi should be started, it deals with location of configuration folder, files, JVM heap and Java System Properties. ```text -java.arg.2=-Xms8g -java.arg.3=-Xmx16g +java.arg.2=-Xms768m +java.arg.3=-Xmx1g ``` -These properties specify the maximum memory that can be allocated to the JVM `-Xmx16g` and the initial memory allocation `-Xms8g`, values of 8g and 16g are used by default, however you may need to change these to fully utilise the memory of the machines you are spinning the service on. +These are the reference profile's initial (`-Xms768m`) and maximum (`-Xmx1g`) +JVM heap sizes. Set `NIFI_JVM_HEAP_INIT` and `NIFI_JVM_HEAP_MAX` in the selected +`nifi.env` to change them. Keep the maximum heap below `NIFI_DOCKER_RAM` and run +`make -C deploy check-config` before starting services.

### `{logback.xml}` @@ -210,7 +223,7 @@ The most useful log sections are: ``` -- This is controls that is logged into the `./nifi/nifi-app.log` +- This sets the default logging level for loggers without a more specific setting. ### `{zookeeper.conf}` diff --git a/security/env/README.md b/security/env/README.md index fd86357a1..52b2703c3 100644 --- a/security/env/README.md +++ b/security/env/README.md @@ -1,14 +1,18 @@ # Development environment defaults The `.env` files in this directory are version-controlled inputs for the -standalone Docker Compose stack and the legacy charts. Their values are public +standalone Docker Compose stack and the owned Helm charts. Their values are public development defaults, not secrets. Never store production credentials in these files. Production deployments must inject credentials through an external secret store or deployment-local configuration that is not committed to Git. -Moving these files to ignored, locally generated `.env` files requires a -coordinated change to Compose, the chart file symlinks, CI, and developer setup. -That migration should be completed separately rather than leaving fresh -checkouts unable to start. +The repository explicitly allows these public defaults in `.gitignore`. +New default files must also be added to that allowlist. OpenSSL templates under +`security/templates/` are tracked source files; generated certificates and keys +remain ignored. + +Put local overrides under `deploy/local/security/` or outside the repository, +then select their parent directory with `COGSTACK_CONFIG_DIR`. See +[deployment configuration](../../docs/deploy/configuration.md#deployment-specific-overrides). From fad904f7ea22e3b586a4e74b76f18ab2f64cd06f Mon Sep 17 00:00:00 2001 From: vladd-bit Date: Thu, 5 Nov 2026 12:53:17 +0000 Subject: [PATCH 21/28] PGSQL updated to 18.6 (kube). --- deploy/charts/postgresql/Chart.yaml | 4 ++-- deploy/charts/postgresql/values.yaml | 2 +- deploy/services.yml | 2 +- scripts/tests/test_configuration.py | 4 +++- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/deploy/charts/postgresql/Chart.yaml b/deploy/charts/postgresql/Chart.yaml index 5d5b96e82..33e07f6cc 100644 --- a/deploy/charts/postgresql/Chart.yaml +++ b/deploy/charts/postgresql/Chart.yaml @@ -2,5 +2,5 @@ apiVersion: v2 name: cogstack-postgresql description: CloudNativePG-backed PostgreSQL cluster for CogStack deployments type: application -version: 0.1.0 -appVersion: "18.1" +version: 0.1.1 +appVersion: "18.6" diff --git a/deploy/charts/postgresql/values.yaml b/deploy/charts/postgresql/values.yaml index 3e5a73bf9..03cc05c6e 100644 --- a/deploy/charts/postgresql/values.yaml +++ b/deploy/charts/postgresql/values.yaml @@ -26,7 +26,7 @@ usersEnvFile: cluster: enabled: true instances: 3 - imageName: ghcr.io/cloudnative-pg/postgresql:18.1-system-trixie@sha256:51bc94428be3df3e234447278e33803544a93d27a57a4b6af737f218d6472d12 + imageName: ghcr.io/cloudnative-pg/postgresql:18.6-system-trixie@sha256:348997438269fddc0e2825e00594367ad61751adb08a7698a0a14f793b0dc2ce imagePullPolicy: IfNotPresent databaseName: "" ownerName: "" diff --git a/deploy/services.yml b/deploy/services.yml index c68a801da..b863e0c12 100644 --- a/deploy/services.yml +++ b/deploy/services.yml @@ -265,7 +265,7 @@ services: #---------------------------------------------------------------------------# cogstack-databank-db: <<: *db-common - image: postgres:18.1-trixie@sha256:1090bc3a8ccfb0b55f78a494d76f8d603434f7e4553543d6e807bc7bd6bbd17f + image: postgres:18.6-trixie@sha256:979bccfde35445db618c65107d27d08c6c21339d8dc3ad4c830c6633947be0e4 container_name: cogstack-production-databank-db platform: linux/amd64 environment: diff --git a/scripts/tests/test_configuration.py b/scripts/tests/test_configuration.py index 653057f54..e6870083c 100755 --- a/scripts/tests/test_configuration.py +++ b/scripts/tests/test_configuration.py @@ -64,7 +64,9 @@ def test_compose_uses_profile_for_resources_ports_and_container_environment(self "NIFI_DATA_PATH": str(self.directory / "data with spaces"), }) selected_value = "fixture-$value with spaces" - self.settings("users_nifi.env", {"NIFI_TEST_VALUE": selected_value}) + users_settings = self.settings("users_nifi.env", {}) + with users_settings.open("a") as settings_file: + settings_file.write(f"\nNIFI_TEST_VALUE='{selected_value}'\n") for compose_file in ("services.yml", "services.dev.yml"): with self.subTest(compose_file=compose_file): result = self.run_bash( From 3fe2eb62cc0659d68e62dc2c37c9b8a984861a69 Mon Sep 17 00:00:00 2001 From: vladd-bit Date: Thu, 5 Nov 2026 17:37:02 +0000 Subject: [PATCH 22/28] Services: PgSQL/MSSQL updates - Each initializer uses one read-only bootstrap directory. - Scripts and SQL remain editable on-premises. - Loaders resolve files relative to their bootstrap directory. - PostgreSQL now uses standard POSTGRES_DB initialization. --- deploy/Makefile | 11 +- deploy/database.env | 3 +- deploy/services.yml | 41 +++-- docs/deploy/services.md | 67 +++++++- scripts/tests/test_mssql_initialization.py | 162 ++++++++++++++++++ scripts/tests/test_postgres_initialization.py | 155 +++++++++++++++++ services/cogstack-db/mssql/init_db.sh | 119 +++++++------ .../schemas/annotations_nlp_create_schema.sql | 125 ++++++++------ .../mssql/schemas/cogstack_db_databank.sql | 2 +- services/cogstack-db/pgsql/init_db.sh | 66 +++---- services/pgsamples/init_db.sh | 78 ++++----- 11 files changed, 629 insertions(+), 200 deletions(-) create mode 100755 scripts/tests/test_mssql_initialization.py create mode 100755 scripts/tests/test_postgres_initialization.py diff --git a/deploy/Makefile b/deploy/Makefile index edcab1cc5..afcb3f3f5 100644 --- a/deploy/Makefile +++ b/deploy/Makefile @@ -452,6 +452,10 @@ start-medcat-trainer: start-production-db: $(WITH_ENV) docker compose -f services.yml ${DC_START_CMD} cogstack-databank-db +start-production-db-mssql: ## Start SQL Server and wait for its schema initialization job + $(WITH_ENV) docker compose -f services.yml $(DC_START_CMD) cogstack-databank-db-mssql && \ + docker compose -f services.yml run --rm --no-deps cogstack-databank-db-mssql-init + start-ocr-services: $(WITH_ENV) docker compose -f ../services/cogstack-platform/apps/ocr-service/docker/docker-compose.base.yml -f ../services/cogstack-platform/apps/ocr-service/docker/docker-compose.prod.yml $(DC_START_CMD) ocr-service ocr-service-text-only @@ -518,7 +522,7 @@ delete-openwebui: ## Remove the Open WebUI container but preserve its data $(WITH_ENV) docker compose -f services.yml rm -f -s open-webui delete-databank: - $(WITH_ENV) docker compose -f services.yml rm -f -s cogstack-databank-db cogstack-databank-db-mssql + $(WITH_ENV) docker compose -f services.yml rm -f -s cogstack-databank-db cogstack-databank-db-mssql-init cogstack-databank-db-mssql delete-databank-volumes: $(WITH_ENV) docker compose -f services.yml $(DC_DOWN_CMD) -v cogstack-databank-db cogstack-databank-db-mssql @@ -624,6 +628,11 @@ stop-ocr-services: stop-production-db: $(WITH_ENV) docker compose -f services.yml $(DC_STOP_CMD) cogstack-databank-db +stop-production-db-mssql: ## Stop the SQL Server initializer and server without removing data + $(WITH_ENV) docker compose -f services.yml $(DC_STOP_CMD) cogstack-databank-db-mssql-init cogstack-databank-db-mssql + +.PHONY: start-production-db-mssql stop-production-db-mssql + stop-data-infra: stop-nifi stop-elastic stop-samples ## Stop core data infrastructure stop-all: stop-data-infra stop-jupyter stop-medcat-service stop-ocr-services ## Stop all major services diff --git a/deploy/database.env b/deploy/database.env index 00052a62a..20bd3f742 100644 --- a/deploy/database.env +++ b/deploy/database.env @@ -1,4 +1,5 @@ POSTGRES_DB_MAX_CONNECTIONS=100 +POSTGRES_SAMPLES_DB_NAME=db_samples # Prefix of file names to load the DB schema for in /services/cogstack-db/(pgsql/mssql)/schemas/ folder POSTGRES_DB_SCHEMA_PREFIX="cogstack_db" @@ -10,7 +11,7 @@ DATABASE_DOCKER_SHM_SIZE=1g DATABASE_DOCKER_CPU_MIN=1 DATABASE_DOCKER_CPU_MAX=1 -DATABASE_DOCKER_RAM=1g +DATABASE_DOCKER_RAM=2g # ClickHouse analytics/ingestion database. These credentials are suitable for # local development only and must be overridden in shared or production use. diff --git a/deploy/services.yml b/deploy/services.yml index b863e0c12..86e35cdc9 100644 --- a/deploy/services.yml +++ b/deploy/services.yml @@ -245,11 +245,10 @@ services: # PG env vars - POSTGRES_USER=${POSTGRES_USER_SAMPLES:-test} - POSTGRES_PASSWORD=${POSTGRES_PASSWORD_SAMPLES:-test} + - POSTGRES_DB=${POSTGRES_SAMPLES_DB_NAME:-db_samples} volumes: - # mapping postgres data dump and initialization - - ../services/pgsamples/db_dump:/data/db_dump:rw - - ../services/pgsamples/schemas:/data/schemas:rw - - ../services/pgsamples/init_db.sh:/docker-entrypoint-initdb.d/init_db.sh:ro + # Editable bootstrap script, schemas, and dumps share one read-only mount. + - ../services/pgsamples:/docker-entrypoint-initdb.d:ro # data persistence - samples-vol:/var/lib/postgresql command: postgres -c "max_connections=${POSTGRES_DB_MAX_CONNECTIONS:-100}" @@ -271,11 +270,9 @@ services: environment: - POSTGRES_USER=${DATABASE_USER:-admin} - POSTGRES_PASSWORD=${DATABASE_PASSWORD:-admin} - - POSTGRES_DATABANK_DB=${DATABASE_DB_NAME:-cogstack} + - POSTGRES_DB=${DATABASE_DB_NAME:-cogstack} volumes: - # mapping postgres data dump and initialization - - ../services/cogstack-db/pgsql/schemas:/data/:ro - - ../services/cogstack-db/pgsql/init_db.sh:/docker-entrypoint-initdb.d/init_db.sh:ro + - ../services/cogstack-db/pgsql:/docker-entrypoint-initdb.d:ro # data persistence - databank-vol:/var/lib/postgresql command: postgres -c "max_connections=${POSTGRES_DB_MAX_CONNECTIONS:-100}" @@ -288,22 +285,33 @@ services: cogstack-databank-db-mssql: <<: *db-common - image: mcr.microsoft.com/mssql/server:2019-latest@sha256:07d7dedecdbfcf6776035dc5eefb5bfd9aa1804b2f6b99f7dedb397d8a599069 + image: &mssql-image mcr.microsoft.com/mssql/server:2019-latest@sha256:07d7dedecdbfcf6776035dc5eefb5bfd9aa1804b2f6b99f7dedb397d8a599069 container_name: cogstack-production-databank-db-mssql environment: - ACCEPT_EULA=y volumes: - # mapping postgres data dump and initialization - - ../services/cogstack-db/mssql/schemas:/data/:ro - - ../services/cogstack-db/mssql/init_db.sh:/usr/src/app/init_db.sh:ro - # data persistence - - databank-vol:/var/opt/mssql - entrypoint: "nohup bash -c '/usr/src/app/init_db.sh'" + # Keep SQL Server files separate from the PostgreSQL databank volume. + - databank-mssql-vol:/var/opt/mssql ports: - 1433:1433 networks: - cognet + cogstack-databank-db-mssql-init: + image: *mssql-image + restart: "no" + env_file: *db-env + environment: + MSSQL_SERVER: cogstack-databank-db-mssql,1433 + depends_on: + cogstack-databank-db-mssql: + condition: service_started + volumes: + - ../services/cogstack-db/mssql:/bootstrap:ro + entrypoint: ["/bin/bash", "/bootstrap/init_db.sh"] + networks: + - cognet + #---------------------------------------------------------------------------# # ClickHouse analytics database # #---------------------------------------------------------------------------# @@ -748,6 +756,9 @@ volumes: databank-vol: driver: local + databank-mssql-vol: + driver: local + # ClickHouse database files clickhouse-data: driver: local diff --git a/docs/deploy/services.md b/docs/deploy/services.md index eaa9ca5f2..be76508cb 100644 --- a/docs/deploy/services.md +++ b/docs/deploy/services.md @@ -189,6 +189,12 @@ Demo dataset with: - cleaned reports - annotation tables +The bootstrap script, schemas, and compressed dumps are supplied together through +one read-only mount of `services/pgsamples/` at `/docker-entrypoint-initdb.d`. +The loader resolves its input paths relative to that directory. Set +`POSTGRES_SAMPLES_DB_NAME` in `deploy/database.env` to choose the sample database +name; Compose passes it to the official image as `POSTGRES_DB`. + ### Access - `localhost:5554` @@ -209,13 +215,72 @@ Local development credentials are defined in `security/env/users_database.env`. Empty database for production ingestion pipelines. Supports both PostgreSQL and MSSQL. -Place schema files inside and they will be loaded instantly on container startup: +Place schema files inside: ```bash services/cogstack-db//schemas/ ``` Where `` can be `mssql` or `pgsql`. + +PostgreSQL applies its initialization SQL only when its data volume is first +created. Compose sets `POSTGRES_DB` from `DATABASE_DB_NAME`, and the official +PostgreSQL entrypoint creates the database before running the bundled loader. +SQL Server runs as the main process in its own container; a separate +`cogstack-databank-db-mssql-init` client job waits for a successful query, creates +the configured database if needed, applies `cogstack_db*.sql` files in filename +order, and then applies `annotations_nlp_create_schema.sql`. SQL errors fail the +job instead of being hidden by a sleeping server wrapper. + +Start SQL Server and wait for initialization to finish with: + +```bash +make -C deploy start-production-db-mssql +``` + +Use `make -C deploy stop-production-db-mssql` to stop it without removing data. +For direct Compose use, load `deploy/export_env_vars.sh` and start both +`cogstack-databank-db-mssql` and `cogstack-databank-db-mssql-init`. Starting only +the server does not run the initializer. + +The job uses `DATABASE_DB_NAME` and `POSTGRES_DB_SCHEMA_PREFIX` from the shared +database configuration. Its readiness wait defaults to 60 attempts, two seconds +apart (each connection attempt has a five-second login timeout). Override +`MSSQL_INIT_RETRIES` and `MSSQL_INIT_RETRY_DELAY_SECONDS` in the initializer's +environment if necessary. The bundled tables use the `CogStack` SQL schema +within the selected database. The job accepts the reference server's self-signed +TLS certificate; adapt client trust settings when supplying a managed server +certificate. + +Bundled MSSQL initialization preserves existing tables and data on repeat runs. +It does not alter existing column definitions. Apply explicit migrations when +updating an existing schema, and make any site-specific initialization SQL safe +to rerun. SQL files stay mounted read-only so on-premises operators can maintain +them independently of the database image. + +Each initializer uses one read-only bootstrap directory: + +| Service | Host directory | Container directory | +|---|---|---| +| `samples-db` | `services/pgsamples/` | `/docker-entrypoint-initdb.d` | +| `cogstack-databank-db` | `services/cogstack-db/pgsql/` | `/docker-entrypoint-initdb.d` | +| `cogstack-databank-db-mssql-init` | `services/cogstack-db/mssql/` | `/bootstrap` | + +The MSSQL job executes `/bootstrap/init_db.sh` and reads `/bootstrap/schemas/` by +default. PostgreSQL reads its loader's sibling `schemas/` directory, plus +`db_dump/` for samples. These directories stay editable on the host. Updating +mounted SQL does not automatically migrate existing PostgreSQL databases. + +### Storage and memory + +- PostgreSQL retains the existing `databank-vol` volume. +- SQL Server uses its own `databank-mssql-vol` volume at `/var/opt/mssql`. +- The shared `DATABASE_DOCKER_RAM` default is `2g`, including the sample database. + +Existing deployments that stored MSSQL files in the formerly shared +`databank-vol` must back up and restore their MSSQL databases into the dedicated +volume before switching. No existing volumes are automatically moved or deleted. + ### Credentials PostgreSQL and MSSQL development credentials are defined in `security/env/users_database.env`. Production credentials must be injected through deployment-local configuration or a secret manager. diff --git a/scripts/tests/test_mssql_initialization.py b/scripts/tests/test_mssql_initialization.py new file mode 100755 index 000000000..9044520d3 --- /dev/null +++ b/scripts/tests/test_mssql_initialization.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 +"""Check readiness, SQL failures, and quoting in the MSSQL initialization job.""" + +import json +import os +import shutil +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +INITIALIZER = ROOT / "services/cogstack-db/mssql/init_db.sh" + + +class MssqlInitializationTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory(prefix="cogstack-mssql-init-") + self.addCleanup(self.temp.cleanup) + self.directory = Path(self.temp.name) + self.schemas = self.directory / "site schemas" + self.schemas.mkdir() + self.annotations = self.schemas / "annotations_nlp_create_schema.sql" + self.annotations.write_text("-- annotations fixture\n") + (self.schemas / "cogstack_db_01 first.sql").write_text("-- custom fixture\n") + (self.schemas / "cogstack_db_02 second.sql").write_text("-- custom fixture\n") + self.log = self.directory / "calls.jsonl" + self.sqlcmd = self.directory / "sqlcmd" + self.sqlcmd.write_text( + f"#!{sys.executable}\n" + + """ +import json +import os +import sys +from pathlib import Path + +args = sys.argv[1:] +log = Path(os.environ["FAKE_SQLCMD_LOG"]) +calls = [json.loads(line) for line in log.read_text().splitlines()] if log.exists() else [] +with log.open("a") as stream: + stream.write(json.dumps({ + "args": args, + "password_from_env": os.environ.get("SQLCMDPASSWORD") == "fixture-password", + }) + "\\n") +if "-Q" in args and "SELECT 1" in args[args.index("-Q") + 1]: + attempts = sum("-Q" in c["args"] and "SELECT 1" in c["args"][c["args"].index("-Q") + 1] for c in calls) + if attempts < int(os.environ.get("FAKE_NOT_READY", "0")): + sys.exit(1) +if "-Q" in args and "CREATE DATABASE" in args[args.index("-Q") + 1] and os.environ.get("FAKE_CREATE_FAILURE"): + sys.exit(7) +if "-i" in args and os.environ.get("FAKE_FAIL_FILE") == Path(args[args.index("-i") + 1]).name: + sys.exit(7) +""" + ) + self.sqlcmd.chmod(0o755) + self.environment = { + **os.environ, + "MSSQL_SA_PASSWORD": "fixture-password", + "DATABASE_DB_NAME": "site_databank", + "POSTGRES_DB_SCHEMA_PREFIX": "cogstack_db", + "MSSQL_SCHEMA_DIR": str(self.schemas), + "MSSQL_INIT_RETRIES": "3", + "MSSQL_INIT_RETRY_DELAY_SECONDS": "0", + "SQLCMD_BIN": str(self.sqlcmd), + "FAKE_SQLCMD_LOG": str(self.log), + } + + def run_job(self, script=INITIALIZER, **overrides): + return subprocess.run( + ["bash", str(script)], + env={**self.environment, **overrides}, + text=True, + capture_output=True, + timeout=10, + ) + + def calls(self): + return [json.loads(line) for line in self.log.read_text().splitlines()] if self.log.exists() else [] + + def applied_files(self): + return [ + Path(call["args"][call["args"].index("-i") + 1]).name + for call in self.calls() + if "-i" in call["args"] + ] + + def test_waits_until_ready_then_applies_sql_in_order_and_exits(self): + result = self.run_job(FAKE_NOT_READY="2") + self.assertEqual(result.returncode, 0, result.stderr) + calls = self.calls() + self.assertEqual(len(calls), 7) + self.assertEqual( + self.applied_files(), + ["cogstack_db_01 first.sql", "cogstack_db_02 second.sql", self.annotations.name], + ) + for call in calls: + self.assertIn("-b", call["args"]) + self.assertNotIn("-P", call["args"]) + self.assertTrue(call["password_from_env"]) + + def test_readiness_timeout_does_not_create_or_modify_schema(self): + result = self.run_job(FAKE_NOT_READY="99") + self.assertNotEqual(result.returncode, 0) + self.assertIn("not ready after 3 attempts", result.stderr) + self.assertEqual(len(self.calls()), 3) + self.assertEqual(self.applied_files(), []) + + def test_database_creation_failure_stops_before_schema_changes(self): + result = self.run_job(FAKE_CREATE_FAILURE="1") + self.assertEqual(result.returncode, 7) + self.assertEqual(self.applied_files(), []) + + def test_custom_sql_failure_prevents_later_scripts(self): + result = self.run_job(FAKE_FAIL_FILE="cogstack_db_01 first.sql") + self.assertEqual(result.returncode, 7) + self.assertEqual(self.applied_files(), ["cogstack_db_01 first.sql"]) + + def test_annotation_sql_failure_is_reported_by_job(self): + result = self.run_job(FAKE_FAIL_FILE=self.annotations.name) + self.assertEqual(result.returncode, 7) + self.assertNotIn("initialization completed", result.stdout) + + def test_database_names_are_quoted_for_queries_and_connection_arguments(self): + name = "ward's]notes" + result = self.run_job(DATABASE_DB_NAME=name) + self.assertEqual(result.returncode, 0, result.stderr) + calls = self.calls() + creation = calls[1]["args"] + query = creation[creation.index("-Q") + 1] + self.assertIn("DB_ID(N'ward''s]notes')", query) + self.assertIn("CREATE DATABASE [ward's]]notes]", query) + for call in calls[2:]: + self.assertEqual(call["args"][call["args"].index("-d") + 1], name) + + def test_missing_password_fails_before_connecting(self): + result = self.run_job(MSSQL_SA_PASSWORD="") + self.assertNotEqual(result.returncode, 0) + self.assertIn("Set MSSQL_SA_PASSWORD", result.stderr) + self.assertEqual(self.calls(), []) + + def test_missing_annotation_schema_fails_before_connecting(self): + self.annotations.unlink() + result = self.run_job() + self.assertNotEqual(result.returncode, 0) + self.assertIn("Missing annotation schema", result.stderr) + self.assertEqual(self.calls(), []) + + def test_schema_directory_defaults_to_the_mounted_bootstrap_bundle(self): + script = self.directory / "init_db.sh" + shutil.copy2(INITIALIZER, script) + shutil.copytree(self.schemas, self.directory / "schemas") + result = self.run_job(script=script, MSSQL_SCHEMA_DIR="") + self.assertEqual(result.returncode, 0, result.stderr) + for call in self.calls(): + if "-i" in call["args"]: + path = Path(call["args"][call["args"].index("-i") + 1]) + self.assertEqual(path.parent, self.directory / "schemas") + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tests/test_postgres_initialization.py b/scripts/tests/test_postgres_initialization.py new file mode 100755 index 000000000..ba17944d0 --- /dev/null +++ b/scripts/tests/test_postgres_initialization.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python3 +"""Exercise the PostgreSQL bootstrap bundles without touching a running database.""" + +import gzip +import json +import os +import shutil +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] + + +class PostgresInitializationTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory(prefix="cogstack-postgres-init-") + self.addCleanup(self.temp.cleanup) + self.directory = Path(self.temp.name) + self.bundle = self.directory / "site bootstrap" + self.bundle.mkdir() + self.schemas = self.bundle / "schemas" + self.schemas.mkdir() + self.dumps = self.bundle / "db_dump" + self.dumps.mkdir() + self.script = self.bundle / "init_db.sh" + self.log = self.directory / "calls.jsonl" + bin_dir = self.directory / "bin" + bin_dir.mkdir() + psql = bin_dir / "psql" + psql.write_text( + f"#!{sys.executable}\n" + + """ +import json +import os +import sys +from pathlib import Path + +args = sys.argv[1:] +stdin = "" if "-f" in args else sys.stdin.read() +with Path(os.environ["FAKE_PSQL_LOG"]).open("a") as stream: + stream.write(json.dumps({"args": args, "stdin": stdin}) + "\\n") +if "-f" in args and os.environ.get("FAKE_FAIL_FILE") == Path(args[args.index("-f") + 1]).name: + sys.exit(7) +""" + ) + psql.chmod(0o755) + self.environment = { + **os.environ, + "PATH": str(bin_dir) + os.pathsep + os.environ["PATH"], + "POSTGRES_USER": "site user", + "POSTGRES_DB": "site database", + "POSTGRES_DB_SCHEMA_PREFIX": "cogstack_db", + "FAKE_PSQL_LOG": str(self.log), + } + + def install_samples_bundle(self): + shutil.copy2(ROOT / "services/pgsamples/init_db.sh", self.script) + (self.schemas / "samples_db_schema.sql").write_text("-- sample schema\n") + for name in ("encounters_pdf_text_small.sql", "encounters_docx_small.sql"): + (self.dumps / name).write_text("-- " + name + "\n") + for name in ( + "samples_db_data.sql.gz", + "encounters_jpg_small.sql.gz", + "encounters_pdf_img_small.sql.gz", + ): + (self.dumps / name).write_bytes(gzip.compress(("-- " + name + "\n").encode())) + + def install_databank_bundle(self): + shutil.copy2(ROOT / "services/cogstack-db/pgsql/init_db.sh", self.script) + for name in ( + "annotations_nlp_create_schema.sql", + "cogstack_db_02 second.sql", + "cogstack_db_01 first.sql", + ): + (self.schemas / name).write_text("-- " + name + "\n") + (self.schemas / "unrelated.sql").write_text("-- should not run\n") + + def run_loader(self, **overrides): + return subprocess.run( + ["bash", str(self.script)], + env={**self.environment, **overrides}, + text=True, + capture_output=True, + timeout=10, + ) + + def calls(self): + return [json.loads(line) for line in self.log.read_text().splitlines()] if self.log.exists() else [] + + def test_samples_load_all_inputs_from_the_bundle_into_postgres_db(self): + self.install_samples_bundle() + result = self.run_loader() + self.assertEqual(result.returncode, 0, result.stderr) + calls = self.calls() + self.assertEqual(len(calls), 6) + self.assertEqual(Path(calls[0]["args"][-1]), self.schemas / "samples_db_schema.sql") + self.assertIn("samples_db_data.sql.gz", calls[1]["stdin"]) + self.assertEqual(Path(calls[2]["args"][-1]).name, "encounters_pdf_text_small.sql") + self.assertEqual(Path(calls[3]["args"][-1]).name, "encounters_docx_small.sql") + self.assertIn("encounters_jpg_small.sql.gz", calls[4]["stdin"]) + self.assertIn("encounters_pdf_img_small.sql.gz", calls[5]["stdin"]) + for call in calls: + self.assertEqual(call["args"][call["args"].index("-d") + 1], "site database") + self.assertEqual(call["args"][call["args"].index("-U") + 1], "site user") + self.assertIn("ON_ERROR_STOP=1", call["args"]) + self.assertNotIn("CREATE DATABASE", call["stdin"]) + + def test_missing_sample_input_fails_before_modifying_the_database(self): + self.install_samples_bundle() + (self.dumps / "samples_db_data.sql.gz").unlink() + result = self.run_loader() + self.assertNotEqual(result.returncode, 0) + self.assertIn("Missing sample database input", result.stderr) + self.assertEqual(self.calls(), []) + + def test_corrupt_compressed_input_stops_the_sample_load(self): + self.install_samples_bundle() + (self.dumps / "samples_db_data.sql.gz").write_bytes(b"not gzip data") + result = self.run_loader() + self.assertNotEqual(result.returncode, 0) + self.assertEqual(len(self.calls()), 2) + self.assertNotIn("initialization completed", result.stdout) + + def test_sample_schema_error_prevents_data_import(self): + self.install_samples_bundle() + result = self.run_loader(FAKE_FAIL_FILE="samples_db_schema.sql") + self.assertEqual(result.returncode, 7) + self.assertEqual(len(self.calls()), 1) + + def test_databank_selects_custom_sql_in_order_without_creating_database(self): + self.install_databank_bundle() + result = self.run_loader() + self.assertEqual(result.returncode, 0, result.stderr) + calls = self.calls() + self.assertEqual( + [Path(c["args"][-1]).name for c in calls], + ["annotations_nlp_create_schema.sql", "cogstack_db_01 first.sql", "cogstack_db_02 second.sql"], + ) + for call in calls: + self.assertEqual(Path(call["args"][-1]).parent, self.schemas) + self.assertEqual(call["args"][call["args"].index("-d") + 1], "site database") + self.assertNotIn("CREATE DATABASE", call["stdin"]) + + def test_databank_sql_error_prevents_later_files(self): + self.install_databank_bundle() + result = self.run_loader(FAKE_FAIL_FILE="cogstack_db_01 first.sql") + self.assertEqual(result.returncode, 7) + self.assertEqual(len(self.calls()), 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/services/cogstack-db/mssql/init_db.sh b/services/cogstack-db/mssql/init_db.sh index fe68d1863..bf25a8143 100755 --- a/services/cogstack-db/mssql/init_db.sh +++ b/services/cogstack-db/mssql/init_db.sh @@ -1,58 +1,81 @@ -#!/bin/bash +#!/usr/bin/env bash +# Run as a one-shot client job; the SQL Server container owns the server process. +set -euo pipefail +server="${MSSQL_SERVER:-cogstack-databank-db-mssql,1433}" +db_user="${MSSQL_SA_USER:-sa}" +db_name="${DATABASE_DB_NAME:-CogStack}" +bootstrap_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +data_dir="${MSSQL_SCHEMA_DIR:-$bootstrap_dir/schemas}" +schema_prefix="${POSTGRES_DB_SCHEMA_PREFIX:-cogstack_db}" +annotations_schema="$data_dir/annotations_nlp_create_schema.sql" +retries="${MSSQL_INIT_RETRIES:-60}" +retry_delay="${MSSQL_INIT_RETRY_DELAY_SECONDS:-2}" +sqlcmd_bin="${SQLCMD_BIN:-}" -# this script is a hack to work around the fact that MSSQL images don't have a predefined startup script point like PGSQL -# https://github.com/microsoft/mssql-docker/issues/2 +: "${MSSQL_SA_PASSWORD:?Set MSSQL_SA_PASSWORD before initializing SQL Server}" +export SQLCMDPASSWORD="$MSSQL_SA_PASSWORD" -# start up mssql-server - -process_id=$(pgrep sqlservr) - -if [[ -z ${process_id} ]]; then - echo "Starting mssql-server..." - /opt/mssql/bin/sqlservr & - sleep 20s -else - echo "mssql-server already started, PID:"$process_id +if [[ ! "$retries" =~ ^[1-9][0-9]*$ || ! "$retry_delay" =~ ^[0-9]+$ ]]; then + echo "MSSQL_INIT_RETRIES must be positive and MSSQL_INIT_RETRY_DELAY_SECONDS non-negative integers." >&2 + exit 1 +fi +if [[ -z "$db_name" || ${#db_name} -gt 128 ]]; then + echo "DATABASE_DB_NAME must contain between 1 and 128 characters." >&2 + exit 1 +fi +if [[ ! -r "$annotations_schema" ]]; then + echo "Missing annotation schema: $annotations_schema" >&2 + exit 1 fi -# global variables -# -DATABANK_DB="CogStack" -DATA_DIR="/data" -ANNOTATIONS_NLP_DB_SCHEMA_FILE="annotations_nlp_create_schema.sql" - -# database schemas that you wish to be created -POSTGRES_DB_SCHEMA_PREFIX="cogstack_db" +if [[ -z "$sqlcmd_bin" ]]; then + for candidate in /opt/mssql-tools18/bin/sqlcmd /opt/mssql-tools/bin/sqlcmd; do + if [[ -x "$candidate" ]]; then + sqlcmd_bin="$candidate" + break + fi + done +fi +if [[ ! -x "$sqlcmd_bin" ]]; then + echo "sqlcmd was not found. Install mssql-tools or set SQLCMD_BIN to its executable path." >&2 + exit 1 +fi -# create the user, the database and set up the access -# -echo "Creating database: $DATABANK_DB and user: $MSSQL_SA_USER" -/opt/mssql-tools/bin/sqlcmd -S "localhost,1433" -U $MSSQL_SA_USER -P $MSSQL_SA_PASSWORD -Q " -IF NOT EXISTS(SELECT * FROM sys.databases WHERE name = '$DATABANK_DB') -BEGIN - CREATE DATABASE [$DATABANK_DB] -END -GO - USE [$DATABANK_DB] -" - -# custom db schema -echo "Creating custom databank DB schemas..." -file_paths=$(find $DATA_DIR/ -name "$POSTGRES_DB_SCHEMA_PREFIX*") - -for file_path in $file_paths; do - echo "importing : "$file_path; - /opt/mssql-tools/bin/sqlcmd -S localhost -U $MSSQL_SA_USER -P $MSSQL_SA_PASSWORD -d $DATABANK_DB -i $file_path; +# The reference Compose server uses its self-signed certificate. -b propagates +# SQL errors to the job exit status; the password stays out of command arguments. +sqlcmd=("$sqlcmd_bin" -S "$server" -U "$db_user" -C -b -l 5) +for ((attempt = 1; attempt <= retries; attempt++)); do + if "${sqlcmd[@]}" -d master -Q 'SET NOCOUNT ON; SELECT 1;' >/dev/null 2>&1; then + break + fi + if ((attempt == retries)); then + echo "SQL Server at $server was not ready after $retries attempts." >&2 + exit 1 + fi + echo "Waiting for SQL Server at $server ($attempt/$retries)..." + sleep "$retry_delay" done -echo "Done." -# create schemas -# -echo "Defining ANNOTATION DB schemas..." -/opt/mssql-tools/bin/sqlcmd -S localhost -U $MSSQL_SA_USER -P $MSSQL_SA_PASSWORD -d $DATABANK_DB -i $DATA_DIR/$ANNOTATIONS_NLP_DB_SCHEMA_FILE -echo "Done." +# Quote database names independently as a SQL string and a delimited identifier. +single_quote="'" +db_literal="${db_name//$single_quote/$single_quote$single_quote}" +db_identifier="${db_name//]/]]}" +echo "Initializing database: $db_name" +"${sqlcmd[@]}" -d master -Q " +IF DB_ID(N'$db_literal') IS NULL +BEGIN + CREATE DATABASE [$db_identifier]; +END;" -while :; do - sleep 10 +# Shell glob order is deterministic and quoted arrays preserve spaces in paths. +shopt -s nullglob +schema_files=("$data_dir"/"$schema_prefix"*.sql) +for schema_file in "${schema_files[@]}"; do + echo "Applying schema: $schema_file" + "${sqlcmd[@]}" -d "$db_name" -i "$schema_file" done + +echo "Applying annotation schema: $annotations_schema" +"${sqlcmd[@]}" -d "$db_name" -i "$annotations_schema" +echo "SQL Server database initialization completed." diff --git a/services/cogstack-db/mssql/schemas/annotations_nlp_create_schema.sql b/services/cogstack-db/mssql/schemas/annotations_nlp_create_schema.sql index c2e378b67..df0bc5b2f 100644 --- a/services/cogstack-db/mssql/schemas/annotations_nlp_create_schema.sql +++ b/services/cogstack-db/mssql/schemas/annotations_nlp_create_schema.sql @@ -1,58 +1,83 @@ -USE [CogStack] -SET ANSI_NULLS ON +-- The initializer selects DATABASE_DB_NAME. CogStack below is the SQL schema, +-- independently of the configured database name. +SET ANSI_NULLS ON; +SET QUOTED_IDENTIFIER ON; +SET XACT_ABORT ON; +BEGIN TRANSACTION; GO -SET QUOTED_IDENTIFIER ON +IF SCHEMA_ID(N'CogStack') IS NULL + EXEC(N'CREATE SCHEMA [CogStack] AUTHORIZATION [dbo]'); GO -CREATE TABLE [CogStack].[documents] ( - id uniqueidentifier, - document_id VARCHAR NOT NULL, - document_text TEXT, - CONSTRAINT PK_documents PRIMARY KEY (id) -); +IF OBJECT_ID(N'CogStack.documents', N'U') IS NULL +BEGIN + CREATE TABLE [CogStack].[documents] ( + id UNIQUEIDENTIFIER NOT NULL, + document_id NVARCHAR(MAX) NOT NULL, + document_text NVARCHAR(MAX) NULL, + CONSTRAINT PK_documents PRIMARY KEY (id) + ); +END; +GO -CREATE TABLE [CogStack].[nlp_models] ( - id BIGINT, - name VARCHAR NOT NULL, - tag VARCHAR, - description VARCHAR, - domains VARCHAR, - url VARCHAR, - CONSTRAINT PK_nlp_models PRIMARY KEY (id) -); +IF OBJECT_ID(N'CogStack.nlp_models', N'U') IS NULL +BEGIN + CREATE TABLE [CogStack].[nlp_models] ( + id BIGINT NOT NULL, + [name] NVARCHAR(MAX) NOT NULL, + tag NVARCHAR(MAX) NULL, + [description] NVARCHAR(MAX) NULL, + domains NVARCHAR(MAX) NULL, + [url] NVARCHAR(MAX) NULL, + CONSTRAINT PK_nlp_models PRIMARY KEY (id) + ); +END; +GO +IF OBJECT_ID(N'CogStack.annotations', N'U') IS NULL +BEGIN + CREATE TABLE [CogStack].[annotations] ( + id BIGINT NOT NULL, + elasticsearch_id NVARCHAR(MAX) NULL, + label NVARCHAR(255) NOT NULL, + label_id NVARCHAR(10) NULL, + source_value NVARCHAR(MAX) NULL, + accuracy DECIMAL(18, 10) NULL, + context_similarity DECIMAL(18, 10) NULL, + star_char INTEGER NULL, + end_char INTEGER NULL, + medcat_info NVARCHAR(MAX) NULL, + tui NVARCHAR(20) NULL, + cui NVARCHAR(20) NULL, + icd10 NVARCHAR(MAX) NULL, + ontologies NVARCHAR(MAX) NULL, + snomed NVARCHAR(MAX) NULL, + [type] NVARCHAR(255) NULL, + medcat_version NVARCHAR(MAX) NULL, + model_id_used BIGINT NULL, + CONSTRAINT PK_annotations PRIMARY KEY (id), + CONSTRAINT FK_nlp_models_annotations FOREIGN KEY (model_id_used) + REFERENCES [CogStack].[nlp_models] (id) + ); +END; +GO -CREATE TABLE [CogStack].[annotations] ( - id BIGINT, - elasticsearch_id VARCHAR NULL, - label VARCHAR NOT NULL, - lael_id VARCHAR, - source_value VARCHAR, - accuracy DECIMAL, - context_similarity DECIMAL, - star_char INTEGER, - end_char INTEGER, - medcat_info VARCHAR, - tui VARCHAR, - cui VARCHAR, - icd10 VARCHAR, - ontologies VARCHAR, - snomed VARCHAR, - "type" VARCHAR, - medcat_version VARCHAR, - model_id_used INTEGER, - CONSTRAINT PK_annotations PRIMARY KEY (id), - CONSTRAINT FK_nlp_models_annotations FOREIGN KEY (model_id_used) REFERENCES [CogStack].[nlp_models] (id) -); +IF OBJECT_ID(N'CogStack.meta_annotations', N'U') IS NULL +BEGIN + CREATE TABLE [CogStack].[meta_annotations] ( + id BIGINT NOT NULL, + annotation_id BIGINT NULL, + [value] NVARCHAR(MAX) NULL, + confidence DECIMAL(18, 10) NULL, + meta_annotation_value NVARCHAR(MAX) NULL, + [name] NVARCHAR(MAX) NULL, + CONSTRAINT PK_meta_annotations PRIMARY KEY (id), + CONSTRAINT FK_meta_annotations_annotations FOREIGN KEY (annotation_id) + REFERENCES [CogStack].[annotations] (id) + ); +END; +GO -CREATE TABLE [CogStack].[meta_annotations] ( - id BIGINT, - annotation_id BIGINT REFERENCES annotations, - "value" VARCHAR, - confidence DECIMAL, - meta_annotation_value VARCHAR, - "name" VARCHAR - CONSTRAINT PK_meta_annotations PRIMARY KEY (id), - CONSTRAINT FK_meta_annotations_annotations FOREIGN KEY (annotation_id) REFERENCES [CogStack].[annotations] (id) -); +COMMIT TRANSACTION; +GO diff --git a/services/cogstack-db/mssql/schemas/cogstack_db_databank.sql b/services/cogstack-db/mssql/schemas/cogstack_db_databank.sql index 3fe441df9..f31e506d3 100644 --- a/services/cogstack-db/mssql/schemas/cogstack_db_databank.sql +++ b/services/cogstack-db/mssql/schemas/cogstack_db_databank.sql @@ -1,4 +1,4 @@ -USE [CogStack] +-- The initializer connects to DATABASE_DB_NAME before applying custom SQL. SET ANSI_NULLS ON GO diff --git a/services/cogstack-db/pgsql/init_db.sh b/services/cogstack-db/pgsql/init_db.sh index 429c38acd..9c3a2ffb2 100755 --- a/services/cogstack-db/pgsql/init_db.sh +++ b/services/cogstack-db/pgsql/init_db.sh @@ -1,41 +1,29 @@ -#!/bin/bash - -set -e - - -# global variables -# -POSTGRES_DATABANK_DB=${POSTGRES_DATABANK_DB:-"cogstack"} -POSTGRES_USER=${POSTGRES_USER:-"admin"} -POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-"admin"} -DATA_DIR="/data" -ANNOTATIONS_NLP_DB_SCHEMA_FILE="annotations_nlp_create_schema.sql" - -# database schemas that you wish to be created -POSTGRES_DB_SCHEMA_PREFIX=${POSTGRES_DB_SCHEMA_PREFIX:-"cogstack_db"} - -# create the user, the database and set up the access -# -echo "Creating database: $POSTGRES_DATABANK_DB and user: $POSTGRES_USER" -psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" <<-EOSQL -CREATE DATABASE $POSTGRES_DATABANK_DB; -GRANT ALL PRIVILEGES ON DATABASE $POSTGRES_DATABANK_DB TO $POSTGRES_USER; -EOSQL - -# create schemas -# -echo "Defining ANNOTATION DB schemas" -psql -v ON_ERROR_STOP=1 -U $POSTGRES_USER -d $POSTGRES_DATABANK_DB -f $DATA_DIR/$ANNOTATIONS_NLP_DB_SCHEMA_FILE - -# custom db schema -echo "Creating custom databank DB schemas" - -file_paths=$(find $DATA_DIR/ -name "$POSTGRES_DB_SCHEMA_PREFIX*") - -for file_path in $file_paths; do - psql -v ON_ERROR_STOP=1 -U $POSTGRES_USER -d $POSTGRES_DATABANK_DB -f $file_path; +#!/usr/bin/env bash +# PostgreSQL's entrypoint creates POSTGRES_DB before running this loader. +set -euo pipefail + +bootstrap_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +schema_dir="$bootstrap_dir/schemas" +db_user="${POSTGRES_USER:-admin}" +db_name="${POSTGRES_DB:-cogstack}" +schema_prefix="${POSTGRES_DB_SCHEMA_PREFIX:-cogstack_db}" +annotations_schema="$schema_dir/annotations_nlp_create_schema.sql" + +if [[ ! -r "$annotations_schema" ]]; then + echo "Missing annotation schema: $annotations_schema" >&2 + exit 1 +fi + +psql_command=(psql -v ON_ERROR_STOP=1 -U "$db_user" -d "$db_name") +echo "Defining annotation schemas in $db_name" +"${psql_command[@]}" -f "$annotations_schema" + +# Glob order is deterministic, and quoted arrays preserve spaces in file names. +shopt -s nullglob +schema_files=("$schema_dir"/"$schema_prefix"*.sql) +for schema_file in "${schema_files[@]}"; do + echo "Applying schema: $schema_file" + "${psql_command[@]}" -f "$schema_file" done -# cleanup -# -echo "Done with initializing the sample data." \ No newline at end of file +echo "PostgreSQL databank initialization completed." diff --git a/services/pgsamples/init_db.sh b/services/pgsamples/init_db.sh index 8650c3b47..3cd6bac6f 100755 --- a/services/pgsamples/init_db.sh +++ b/services/pgsamples/init_db.sh @@ -1,54 +1,44 @@ -#!/bin/bash - -set -e - - -# global variables -# -DB_USER=${POSTGRES_USER:-"test"} -DB_USER_PASSWORD=${POSTGRES_PASSWORD_SAMPLES:-"test"} -DB_NAME="db_samples" - -DATA_DIR="/data" -DB_DUMP_FILE="/db_dump/samples_db_data.sql.gz" -DB_SCHEMA_FILE="/schemas/samples_db_schema.sql" - -DB_DUMP_FILES_EXTRA_SQL_FILES=( - $DATA_DIR"/db_dump/encounters_pdf_text_small.sql" - $DATA_DIR"/db_dump/encounters_docx_small.sql" +#!/usr/bin/env bash +# PostgreSQL's entrypoint creates POSTGRES_DB before running this loader. +set -euo pipefail + +bootstrap_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +db_user="${POSTGRES_USER:-test}" +db_name="${POSTGRES_DB:-db_samples}" +schema_file="$bootstrap_dir/schemas/samples_db_schema.sql" +dump_file="$bootstrap_dir/db_dump/samples_db_data.sql.gz" +extra_sql_files=( + "$bootstrap_dir/db_dump/encounters_pdf_text_small.sql" + "$bootstrap_dir/db_dump/encounters_docx_small.sql" ) - -DB_DUMP_FILES_EXTRA_SQL_GZ_FILES=( - $DATA_DIR"/db_dump/encounters_jpg_small.sql.gz" - $DATA_DIR"/db_dump/encounters_pdf_img_small.sql.gz" +extra_gzip_files=( + "$bootstrap_dir/db_dump/encounters_jpg_small.sql.gz" + "$bootstrap_dir/db_dump/encounters_pdf_img_small.sql.gz" ) -# create the user, the database and set up the access -# -echo "Creating database: $DB_NAME and user: $DB_USER" -psql -v ON_ERROR_STOP=1 -U "$DB_USER" <<-EOSQL -CREATE DATABASE $DB_NAME OWNER $DB_USER; -GRANT ALL PRIVILEGES ON DATABASE $DB_NAME TO $DB_USER; -EOSQL +# Check the bundle before making any schema or data changes. +for input_file in "$schema_file" "$dump_file" "${extra_sql_files[@]}" "${extra_gzip_files[@]}"; do + if [[ ! -r "$input_file" ]]; then + echo "Missing sample database input: $input_file" >&2 + exit 1 + fi +done -# create schemas -# -echo "Defining DB schemas: $DB_SCHEMA_FILE" -psql -v ON_ERROR_STOP=1 -U $DB_USER -d $DB_NAME -f $DATA_DIR/$DB_SCHEMA_FILE +psql_command=(psql -v ON_ERROR_STOP=1 -U "$db_user" -d "$db_name") +echo "Defining sample database schemas in $db_name" +"${psql_command[@]}" -f "$schema_file" -echo "Restoring DB from dump: $DB_DUMP_FILE" -gunzip -c $DATA_DIR/$DB_DUMP_FILE | psql -v ON_ERROR_STOP=1 -U $DB_USER -d $DB_NAME +echo "Restoring sample data from $dump_file" +gunzip -c "$dump_file" | "${psql_command[@]}" -for sqlfile in "${DB_DUMP_FILES_EXTRA_SQL_FILES[@]}"; do - echo "=== Applying $sqlfile ===" - psql -v ON_ERROR_STOP=1 -U "$DB_USER" -d "$DB_NAME" -f "$sqlfile" +for sql_file in "${extra_sql_files[@]}"; do + echo "Applying $sql_file" + "${psql_command[@]}" -f "$sql_file" done -for gzfile in "${DB_DUMP_FILES_EXTRA_SQL_GZ_FILES[@]}"; do - echo "=== Applying $gzfile ===" -gunzip -c "$gzfile" | psql -v ON_ERROR_STOP=1 -U $DB_USER -d $DB_NAME +for gzip_file in "${extra_gzip_files[@]}"; do + echo "Restoring $gzip_file" + gunzip -c "$gzip_file" | "${psql_command[@]}" done -# cleanup -# -echo "Done with initializing the sample data." +echo "Sample database initialization completed." From a060df4edfe60cbaacde3cda3bc29b380000ff10 Mon Sep 17 00:00:00 2001 From: vladd-bit Date: Thu, 5 Nov 2026 19:55:26 +0000 Subject: [PATCH 23/28] Services: added TLS to PgSQL. --- .github/workflows/compose_config.yml | 18 ++ .github/workflows/smoke_nifi_services.yml | 4 +- .github/workflows/smoke_postgres_services.yml | 13 ++ deploy/Makefile | 18 +- deploy/database.env | 17 ++ deploy/services.dev.yml | 43 +++- deploy/services.yml | 141 ++++++++++--- docs/deploy/air-gapped.md | 13 +- docs/deploy/configuration.md | 13 ++ docs/deploy/deployment.md | 5 +- docs/deploy/services.md | 23 ++- .../workflows/database_ocr_to_opensearch.md | 11 +- .../workflows/database_to_opensearch.md | 12 +- docs/security/certificates.md | 23 ++- docs/security/databases.md | 145 +++++++++++++ docs/security/elasticsearch_opensearch.md | 4 +- docs/security/main.md | 4 +- docs/security/nifi.md | 7 +- mkdocs.yml | 1 + nifi/user_scripts/dto/database_config.py | 10 +- scripts/tests/smoke_postgres_services.sh | 20 +- .../test_database_deployment_security.py | 152 ++++++++++++++ scripts/tests/test_mssql_initialization.py | 35 +++- scripts/tests/test_opensearch_deployment.py | 8 + scripts/tests/test_postgres_certificates.py | 193 ++++++++++++++++++ scripts/tests/test_postgres_initialization.py | 13 +- security/env/users_database.env | 11 +- .../scripts/create_opensearch_node_cert.sh | 5 +- security/scripts/create_postgres_certs.sh | 72 +++++++ services/cogstack-db/mssql/access/ingest.sql | 71 +++++++ services/cogstack-db/mssql/init_db.sh | 38 +++- .../cogstack-db/pgsql/access/configure.sh | 22 ++ services/cogstack-db/pgsql/access/ingest.sql | 66 ++++++ .../cogstack-db/pgsql/access/pg_hba_tls.conf | 3 + services/cogstack-db/pgsql/init_db.sh | 1 + services/pgsamples/access/configure.sh | 22 ++ services/pgsamples/access/ingest.sql | 63 ++++++ services/pgsamples/access/pg_hba_tls.conf | 5 + services/pgsamples/init_db.sh | 1 + 39 files changed, 1222 insertions(+), 104 deletions(-) create mode 100644 docs/security/databases.md create mode 100755 scripts/tests/test_database_deployment_security.py create mode 100644 scripts/tests/test_postgres_certificates.py create mode 100755 security/scripts/create_postgres_certs.sh create mode 100644 services/cogstack-db/mssql/access/ingest.sql create mode 100755 services/cogstack-db/pgsql/access/configure.sh create mode 100644 services/cogstack-db/pgsql/access/ingest.sql create mode 100644 services/cogstack-db/pgsql/access/pg_hba_tls.conf create mode 100755 services/pgsamples/access/configure.sh create mode 100644 services/pgsamples/access/ingest.sql create mode 100644 services/pgsamples/access/pg_hba_tls.conf diff --git a/.github/workflows/compose_config.yml b/.github/workflows/compose_config.yml index 3d8fa39cf..a8df8a9b6 100644 --- a/.github/workflows/compose_config.yml +++ b/.github/workflows/compose_config.yml @@ -15,6 +15,12 @@ on: - "scripts/tests/check_opensearch_settings.py" - "scripts/tests/test_opensearch_deployment.py" - "services/elasticsearch/**" + - "services/cogstack-db/**" + - "services/pgsamples/**" + - "scripts/tests/test_database_deployment_security.py" + - "scripts/tests/test_postgres_certificates.py" + - "scripts/tests/test_postgres_initialization.py" + - "scripts/tests/test_mssql_initialization.py" - "services/kibana/config/opensearch.yml" - "security/templates/**" - "security/es_roles/opensearch/**" @@ -42,6 +48,12 @@ on: - "scripts/tests/check_opensearch_settings.py" - "scripts/tests/test_opensearch_deployment.py" - "services/elasticsearch/**" + - "services/cogstack-db/**" + - "services/pgsamples/**" + - "scripts/tests/test_database_deployment_security.py" + - "scripts/tests/test_postgres_certificates.py" + - "scripts/tests/test_postgres_initialization.py" + - "scripts/tests/test_mssql_initialization.py" - "services/kibana/config/opensearch.yml" - "security/templates/**" - "security/es_roles/opensearch/**" @@ -105,6 +117,12 @@ jobs: - name: Test configuration profiles and resource guards run: python3 -B -m unittest discover -s scripts/tests -p test_configuration.py -v + - name: Test database bootstrap and credential isolation + run: | + python3 -B -m unittest discover -s scripts/tests -p 'test_*initialization.py' -v + python3 -B scripts/tests/test_database_deployment_security.py + python3 -B scripts/tests/test_postgres_certificates.py + - name: Test OpenSearch AI configuration run: python3 -B -m unittest discover -s scripts/tests -p test_opensearch_ai_init.py -v diff --git a/.github/workflows/smoke_nifi_services.yml b/.github/workflows/smoke_nifi_services.yml index c9ff74cdc..0e7e138a4 100644 --- a/.github/workflows/smoke_nifi_services.yml +++ b/.github/workflows/smoke_nifi_services.yml @@ -50,11 +50,9 @@ jobs: java-version: '21' - name: Generate certificates - working-directory: security/scripts run: | set -euo pipefail - ./create_root_ca_cert.sh - ./create_nifi_certs.sh + make -C deploy init-security-nifi - name: Smoke tests run: | diff --git a/.github/workflows/smoke_postgres_services.yml b/.github/workflows/smoke_postgres_services.yml index 67bc6c05f..972ff4be2 100644 --- a/.github/workflows/smoke_postgres_services.yml +++ b/.github/workflows/smoke_postgres_services.yml @@ -11,6 +11,8 @@ on: - "deploy/config_files.sh" - "deploy/*.env" - "security/env/*.env" + - "security/scripts/**" + - "security/templates/**" - "deploy/services.yml" - "services/pgsamples" - "services/pgsamples/**" @@ -36,6 +38,17 @@ jobs: steps: - uses: actions/checkout@v7 + - name: Set up Java + uses: actions/setup-java@v6 + with: + distribution: 'temurin' + java-version: '21' + + - name: Generate PostgreSQL certificates + run: | + set -euo pipefail + make -C deploy init-security-postgresql + - name: Start samples database run: | set -euo pipefail diff --git a/deploy/Makefile b/deploy/Makefile index afcb3f3f5..d384b77cb 100644 --- a/deploy/Makefile +++ b/deploy/Makefile @@ -127,7 +127,7 @@ init-security-root-ca: ## Generate the shared root CA only when no CA material e (cd "$(SECURITY_SCRIPTS_DIR)" && ./create_root_ca_cert.sh); \ fi -init-security-nifi: init-security-root-ca ## Generate NiFi certificates only when they are missing or incomplete +init-security-nifi: init-security-opensearch init-security-postgresql ## Prepare NiFi certificates and its search/database certificate inputs @set -euo pipefail; \ nifi_dir="$(SECURITY_CERTIFICATES_DIR)/nifi"; \ if [[ -f "$$nifi_dir/nifi.key" \ @@ -173,8 +173,7 @@ init-security-opensearch: init-security-root-ca ## Generate OpenSearch node, cli fi; \ done; \ done; \ - if [[ ! -f "$$opensearch_dir/elastic-stack-ca.crt.pem" \ - || ! -f "$$opensearch_dir/elastic-stack-ca.p12" ]]; then \ + if [[ ! -f "$$opensearch_dir/elastic-stack-ca.crt.pem" ]]; then \ node_certificates_missing=1; \ fi; \ if (( node_certificates_missing )); then \ @@ -238,12 +237,15 @@ init-security-search: ## Generate certificates for ELASTICSEARCH_VERSION from el ;; \ esac -init-security: ## Generate missing NiFi, Gitea, and selected search-backend certificates +init-security-postgresql: init-security-root-ca ## Generate missing PostgreSQL TLS bundles, preserving supplied certificates + @$(WITH_ENV) SECURITY_CERTIFICATES_DIR="$(SECURITY_CERTIFICATES_DIR)" bash "$(SECURITY_SCRIPTS_DIR)/create_postgres_certs.sh" + +init-security: ## Generate missing NiFi, PostgreSQL, Gitea, and selected search-backend certificates @$(MAKE) --no-print-directory init-security-nifi @$(MAKE) --no-print-directory init-security-gitea @$(MAKE) --no-print-directory init-security-search -.PHONY: init-security init-security-root-ca init-security-nifi init-security-gitea init-security-opensearch init-security-elasticsearch init-security-search +.PHONY: init-security init-security-root-ca init-security-nifi init-security-postgresql init-security-gitea init-security-opensearch init-security-elasticsearch init-security-search helm-template-opensearch: ## Render OpenSearch chart using chart defaults plus ./helm/opensearch.values.yaml $(WITH_ENV) helm template $(HELM_OPENSEARCH_RELEASE) $(HELM_OPENSEARCH_CHART) $(HELM_OPENSEARCH_VALUES_ARG) \ @@ -426,7 +428,8 @@ logs-openwebui: ## Follow the Open WebUI logs $(WITH_ENV) docker compose -f services.yml logs --tail=100 -f open-webui start-samples: ## Start the sample database service - $(WITH_ENV) docker compose -f services.yml $(DC_START_CMD) samples-db + $(WITH_ENV) docker compose -f services.yml $(DC_START_CMD) samples-db && \ + docker compose -f services.yml exec -T -e PGHOST=samples-db samples-db /docker-entrypoint-initdb.d/access/configure.sh start-clickhouse: ## Start the ClickHouse ingestion database $(WITH_ENV) docker compose -f services.yml $(DC_START_CMD) clickhouse @@ -450,7 +453,8 @@ start-medcat-trainer: $(WITH_ENV) docker compose -f ../services/cogstack-nlp/medcat-trainer/docker-compose-prod.yml $(DC_START_CMD) medcattrainer nginx solr start-production-db: - $(WITH_ENV) docker compose -f services.yml ${DC_START_CMD} cogstack-databank-db + $(WITH_ENV) docker compose -f services.yml ${DC_START_CMD} cogstack-databank-db && \ + docker compose -f services.yml exec -T -e PGHOST=cogstack-databank-db cogstack-databank-db /docker-entrypoint-initdb.d/access/configure.sh start-production-db-mssql: ## Start SQL Server and wait for its schema initialization job $(WITH_ENV) docker compose -f services.yml $(DC_START_CMD) cogstack-databank-db-mssql && \ diff --git a/deploy/database.env b/deploy/database.env index 20bd3f742..db9646d61 100644 --- a/deploy/database.env +++ b/deploy/database.env @@ -1,5 +1,22 @@ POSTGRES_DB_MAX_CONNECTIONS=100 POSTGRES_SAMPLES_DB_NAME=db_samples +# Paths are relative to deploy/, or may be absolute for supplied certificate bundles. +POSTGRES_SAMPLES_TLS_DIR="${POSTGRES_SAMPLES_TLS_DIR:-../security/certificates/postgresql/samples}" +POSTGRES_DATABANK_TLS_DIR="${POSTGRES_DATABANK_TLS_DIR:-../security/certificates/postgresql/databank}" +POSTGRES_SAMPLES_JDBC_URL="jdbc:postgresql://samples-db:5432/${POSTGRES_SAMPLES_DB_NAME}?sslmode=verify-full&sslrootcert=/run/database-cas/samples-ca.crt" + +# Host access defaults to loopback; containers continue to use service DNS. +POSTGRES_SAMPLES_OUTPUT_HOST=127.0.0.1 +POSTGRES_SAMPLES_OUTPUT_PORT=5554 +POSTGRES_DATABANK_OUTPUT_HOST=127.0.0.1 +POSTGRES_DATABANK_OUTPUT_PORT=5558 +MSSQL_OUTPUT_HOST=127.0.0.1 +MSSQL_OUTPUT_PORT=1433 + +# Local SQL Server uses a self-signed certificate. Set false with a trusted CA +# installed in the initializer image for certificate/hostname verification. +MSSQL_TRUST_SERVER_CERTIFICATE=true +MSSQL_TLS_HOSTNAME="" # Prefix of file names to load the DB schema for in /services/cogstack-db/(pgsql/mssql)/schemas/ folder POSTGRES_DB_SCHEMA_PREFIX="cogstack_db" diff --git a/deploy/services.dev.yml b/deploy/services.dev.yml index f9a507ce9..6804f3828 100644 --- a/deploy/services.dev.yml +++ b/deploy/services.dev.yml @@ -15,11 +15,9 @@ x-all-env: &all-env - "${COGSTACK_GITEA_ENV_FILE:-./gitea.env}" - "${COGSTACK_NGINX_ENV_FILE:-./nginx.env}" - "${COGSTACK_TELEMETRY_ENV_FILE:-./telemetry.env}" - - "${COGSTACK_DATABASE_ENV_FILE:-./database.env}" - "${COGSTACK_ELASTICSEARCH_ENV_FILE:-./elasticsearch.env}" - "${COGSTACK_NETWORK_SETTINGS_ENV_FILE:-./network_settings.env}" - "${COGSTACK_USERS_NIFI_ENV_FILE:-../security/env/users_nifi.env}" - - "${COGSTACK_USERS_DATABASE_ENV_FILE:-../security/env/users_database.env}" - "${COGSTACK_USERS_NGINX_ENV_FILE:-../security/env/users_nginx.env}" - "${COGSTACK_USERS_ELASTICSEARCH_ENV_FILE:-../security/env/users_elasticsearch.env}" - "${COGSTACK_CERTIFICATES_GENERAL_ENV_FILE:-../security/env/certificates_general.env}" @@ -32,9 +30,16 @@ x-es-env: &es-env - "${COGSTACK_USERS_ELASTICSEARCH_ENV_FILE:-../security/env/users_elasticsearch.env}" - "${COGSTACK_CERTIFICATES_ELASTICSEARCH_ENV_FILE:-../security/env/certificates_elasticsearch.env}" -x-db-env: &db-env - - "${COGSTACK_DATABASE_ENV_FILE:-./database.env}" - - "${COGSTACK_USERS_DATABASE_ENV_FILE:-../security/env/users_database.env}" +x-nifi-db-environment: &nifi-db-environment + POSTGRES_SAMPLES_DB_NAME: ${POSTGRES_SAMPLES_DB_NAME:-db_samples} + POSTGRES_SAMPLES_JDBC_URL: ${POSTGRES_SAMPLES_JDBC_URL:-jdbc:postgresql://samples-db:5432/${POSTGRES_SAMPLES_DB_NAME:-db_samples}?sslmode=verify-full&sslrootcert=/run/database-cas/samples-ca.crt} + POSTGRES_INGEST_USER_SAMPLES: ${POSTGRES_INGEST_USER_SAMPLES:-cogstack_samples} + POSTGRES_INGEST_PASSWORD_SAMPLES: ${POSTGRES_INGEST_PASSWORD_SAMPLES:?Set the sample ingestion password} + DATABASE_DB_NAME: ${DATABASE_DB_NAME:-cogstack} + DATABASE_INGEST_USER: ${DATABASE_INGEST_USER:-cogstack_ingest} + DATABASE_INGEST_PASSWORD: ${DATABASE_INGEST_PASSWORD:?Set the PostgreSQL ingestion password} + MSSQL_INGEST_USER: ${MSSQL_INGEST_USER:-cogstack_ingest} + MSSQL_INGEST_PASSWORD: ${MSSQL_INGEST_PASSWORD:?Set the MSSQL ingestion password} x-common-hosts: &common-hosts - ${ELASTICSEARCH_1_HOST_NAME:-test-1:0.0.0.0} @@ -73,7 +78,25 @@ x-nifi-volumes: &nifi-volumes - ../nifi/user_python_extensions:/opt/nifi/nifi-current/python_extensions:rw # Security certificates - - ../security:/security:ro + - ../security/certificates/nifi:/security/certificates/nifi:ro + - ../security/certificates/elastic/opensearch/elasticsearch:/security/certificates/elastic/opensearch/elasticsearch:ro + - ../security/certificates/elastic/opensearch/elastic-stack-ca.crt.pem:/security/certificates/elastic/opensearch/elastic-stack-ca.crt.pem:ro + - ../security/certificates/elastic/opensearch/es_kibana_client.pem:/security/certificates/elastic/opensearch/es_kibana_client.pem:ro + - ../security/certificates/elastic/opensearch/es_kibana_client.key:/security/certificates/elastic/opensearch/es_kibana_client.key:ro + - ../security/certificates/root/root-ca.pem:/security/certificates/root/root-ca.pem:ro + # Database clients receive only the public CA from each TLS bundle. + - type: bind + source: ${POSTGRES_SAMPLES_TLS_DIR:-../security/certificates/postgresql/samples}/ca.crt + target: /run/database-cas/samples-ca.crt + read_only: true + bind: + create_host_path: false + - type: bind + source: ${POSTGRES_DATABANK_TLS_DIR:-../security/certificates/postgresql/databank}/ca.crt + target: /run/database-cas/databank-ca.crt + read_only: true + bind: + create_host_path: false # Security helper scripts - ../security/scripts/nifi_create_single_user_auth.sh:/opt/nifi/nifi-current/security_scripts/nifi_create_single_user_auth.sh:ro @@ -120,8 +143,9 @@ services: hostname: nifi shm_size: ${NIFI_DOCKER_SHM_SIZE:-"1g"} environment: - - PYTHONPATH=${NIFI_PYTHONPATH:-/opt/nifi/nifi-current/python/framework} - - JVM_OPTS=${NIFI_JVM_OPTS:--XX:+UseG1GC -XX:MaxGCPauseMillis=200 -XX:+ParallelRefProcEnabled -Djava.security.egd=file:/dev/./urandom} + <<: *nifi-db-environment + PYTHONPATH: ${NIFI_PYTHONPATH:-/opt/nifi/nifi-current/python/framework} + JVM_OPTS: ${NIFI_JVM_OPTS:--XX:+UseG1GC -XX:MaxGCPauseMillis=200 -XX:+ParallelRefProcEnabled -Djava.security.egd=file:/dev/./urandom} deploy: resources: limits: @@ -162,7 +186,8 @@ services: volumes: - ../services/nginx/sites-enabled:/etc/nginx/sites-enabled:ro - ../services/nginx/config/nginx.conf.template:/etc/nginx/templates/nginx.conf.template:ro - - ../security/certificates:/certificates:ro + - ../security/certificates/nifi:/certificates/nifi:ro + - ../security/certificates/root/root-ca.pem:/certificates/root/root-ca.pem:ro ports: - "${NIFI_EXTERNAL_PORT_NGINX:-8443}:${NIFI_INTERNAL_PORT_NGINX:-8443}" networks: diff --git a/deploy/services.yml b/deploy/services.yml index 86e35cdc9..ad442b49b 100644 --- a/deploy/services.yml +++ b/deploy/services.yml @@ -26,11 +26,9 @@ x-all-env: &all-env - "${COGSTACK_GITEA_ENV_FILE:-./gitea.env}" - "${COGSTACK_NGINX_ENV_FILE:-./nginx.env}" - "${COGSTACK_TELEMETRY_ENV_FILE:-./telemetry.env}" - - "${COGSTACK_DATABASE_ENV_FILE:-./database.env}" - "${COGSTACK_ELASTICSEARCH_ENV_FILE:-./elasticsearch.env}" - "${COGSTACK_NETWORK_SETTINGS_ENV_FILE:-./network_settings.env}" - "${COGSTACK_USERS_NIFI_ENV_FILE:-../security/env/users_nifi.env}" - - "${COGSTACK_USERS_DATABASE_ENV_FILE:-../security/env/users_database.env}" - "${COGSTACK_USERS_NGINX_ENV_FILE:-../security/env/users_nginx.env}" - "${COGSTACK_USERS_ELASTICSEARCH_ENV_FILE:-../security/env/users_elasticsearch.env}" - "${COGSTACK_CERTIFICATES_GENERAL_ENV_FILE:-../security/env/certificates_general.env}" @@ -43,9 +41,16 @@ x-es-env: &es-env - "${COGSTACK_USERS_ELASTICSEARCH_ENV_FILE:-../security/env/users_elasticsearch.env}" - "${COGSTACK_CERTIFICATES_ELASTICSEARCH_ENV_FILE:-../security/env/certificates_elasticsearch.env}" -x-db-env: &db-env - - "${COGSTACK_DATABASE_ENV_FILE:-./database.env}" - - "${COGSTACK_USERS_DATABASE_ENV_FILE:-../security/env/users_database.env}" +x-nifi-db-environment: &nifi-db-environment + POSTGRES_SAMPLES_DB_NAME: ${POSTGRES_SAMPLES_DB_NAME:-db_samples} + POSTGRES_SAMPLES_JDBC_URL: ${POSTGRES_SAMPLES_JDBC_URL:-jdbc:postgresql://samples-db:5432/${POSTGRES_SAMPLES_DB_NAME:-db_samples}?sslmode=verify-full&sslrootcert=/run/database-cas/samples-ca.crt} + POSTGRES_INGEST_USER_SAMPLES: ${POSTGRES_INGEST_USER_SAMPLES:-cogstack_samples} + POSTGRES_INGEST_PASSWORD_SAMPLES: ${POSTGRES_INGEST_PASSWORD_SAMPLES:?Set the sample ingestion password} + DATABASE_DB_NAME: ${DATABASE_DB_NAME:-cogstack} + DATABASE_INGEST_USER: ${DATABASE_INGEST_USER:-cogstack_ingest} + DATABASE_INGEST_PASSWORD: ${DATABASE_INGEST_PASSWORD:?Set the PostgreSQL ingestion password} + MSSQL_INGEST_USER: ${MSSQL_INGEST_USER:-cogstack_ingest} + MSSQL_INGEST_PASSWORD: ${MSSQL_INGEST_PASSWORD:?Set the MSSQL ingestion password} x-common-hosts: &common-hosts - ${ELASTICSEARCH_1_HOST_NAME:-test-1:0.0.0.0} @@ -87,7 +92,25 @@ x-nifi-volumes: &nifi-volumes - ../nifi/user_python_extensions:/opt/nifi/nifi-current/python_extensions:rw # Security certificates - - ../security:/security:ro + - ../security/certificates/nifi:/security/certificates/nifi:ro + - ../security/certificates/elastic/opensearch/elasticsearch:/security/certificates/elastic/opensearch/elasticsearch:ro + - ../security/certificates/elastic/opensearch/elastic-stack-ca.crt.pem:/security/certificates/elastic/opensearch/elastic-stack-ca.crt.pem:ro + - ../security/certificates/elastic/opensearch/es_kibana_client.pem:/security/certificates/elastic/opensearch/es_kibana_client.pem:ro + - ../security/certificates/elastic/opensearch/es_kibana_client.key:/security/certificates/elastic/opensearch/es_kibana_client.key:ro + - ../security/certificates/root/root-ca.pem:/security/certificates/root/root-ca.pem:ro + # Database clients receive only the public CA from each TLS bundle. + - type: bind + source: ${POSTGRES_SAMPLES_TLS_DIR:-../security/certificates/postgresql/samples}/ca.crt + target: /run/database-cas/samples-ca.crt + read_only: true + bind: + create_host_path: false + - type: bind + source: ${POSTGRES_DATABANK_TLS_DIR:-../security/certificates/postgresql/databank}/ca.crt + target: /run/database-cas/databank-ca.crt + read_only: true + bind: + create_host_path: false # Security helper scripts - ../security/scripts/nifi_create_single_user_auth.sh:/opt/nifi/nifi-current/security_scripts/nifi_create_single_user_auth.sh:ro @@ -116,7 +139,6 @@ x-db-common: &db-common <<: *common-ulimits shm_size: ${DATABASE_DOCKER_SHM_SIZE:-"1g"} restart: unless-stopped - env_file: *db-env deploy: resources: limits: @@ -126,6 +148,36 @@ x-db-common: &db-common cpus: "${DATABASE_DOCKER_CPU_MIN}" memory: "${DATABASE_DOCKER_RAM}" +x-postgres-common: &postgres-common + <<: *db-common + image: postgres:18.6-trixie@sha256:979bccfde35445db618c65107d27d08c6c21339d8dc3ad4c830c6633947be0e4 + platform: linux/amd64 + # Stage host/Secret keys with PostgreSQL ownership before its usual privilege drop. + entrypoint: + - /bin/bash + - -ec + - | + install -d -m 700 -o postgres -g postgres /run/postgresql/tls + install -m 600 -o postgres -g postgres /run/database-tls/server.key /run/postgresql/tls/server.key + install -m 644 -o postgres -g postgres /run/database-tls/server.crt /run/postgresql/tls/server.crt + install -m 644 -o postgres -g postgres /run/database-tls/ca.crt /run/postgresql/tls/ca.crt + exec /usr/local/bin/docker-entrypoint.sh "$$@" + - -- + command: + - postgres + - -c + - max_connections=${POSTGRES_DB_MAX_CONNECTIONS:-100} + - -c + - ssl=on + - -c + - ssl_min_protocol_version=TLSv1.2 + - -c + - ssl_cert_file=/run/postgresql/tls/server.crt + - -c + - ssl_key_file=/run/postgresql/tls/server.key + - -c + - hba_file=/docker-entrypoint-initdb.d/access/pg_hba_tls.conf + x-es-common-volumes: &es-common-volumes # Shared configs - ../services/elasticsearch/search-entrypoint.sh:/usr/local/bin/cogstack-search-entrypoint.sh:ro @@ -237,23 +289,29 @@ services: # Postgres container with sample data # #---------------------------------------------------------------------------# samples-db: - <<: *db-common - image: postgres:18.6-trixie@sha256:979bccfde35445db618c65107d27d08c6c21339d8dc3ad4c830c6633947be0e4 + <<: *postgres-common container_name: cogstack-samples-db - platform: linux/amd64 environment: - # PG env vars - - POSTGRES_USER=${POSTGRES_USER_SAMPLES:-test} - - POSTGRES_PASSWORD=${POSTGRES_PASSWORD_SAMPLES:-test} - - POSTGRES_DB=${POSTGRES_SAMPLES_DB_NAME:-db_samples} + PGSSLMODE: verify-full + PGSSLROOTCERT: /run/postgresql/tls/ca.crt + POSTGRES_USER: ${POSTGRES_USER_SAMPLES:-test} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD_SAMPLES:?Set the sample bootstrap password} + POSTGRES_DB: ${POSTGRES_SAMPLES_DB_NAME:-db_samples} + COGSTACK_INGEST_USER: ${POSTGRES_INGEST_USER_SAMPLES:-cogstack_samples} + COGSTACK_INGEST_PASSWORD: ${POSTGRES_INGEST_PASSWORD_SAMPLES:?Set the sample ingestion password} volumes: # Editable bootstrap script, schemas, and dumps share one read-only mount. - ../services/pgsamples:/docker-entrypoint-initdb.d:ro # data persistence - samples-vol:/var/lib/postgresql - command: postgres -c "max_connections=${POSTGRES_DB_MAX_CONNECTIONS:-100}" + - type: bind + source: ${POSTGRES_SAMPLES_TLS_DIR:-../security/certificates/postgresql/samples} + target: /run/database-tls + read_only: true + bind: + create_host_path: false ports: - - 5554:5432 + - "${POSTGRES_SAMPLES_OUTPUT_HOST:-127.0.0.1}:${POSTGRES_SAMPLES_OUTPUT_PORT:-5554}:5432" expose: - 5432 networks: @@ -263,21 +321,29 @@ services: # CogStack Databank / Cogstack-DB, production database # #---------------------------------------------------------------------------# cogstack-databank-db: - <<: *db-common - image: postgres:18.6-trixie@sha256:979bccfde35445db618c65107d27d08c6c21339d8dc3ad4c830c6633947be0e4 + <<: *postgres-common container_name: cogstack-production-databank-db - platform: linux/amd64 environment: - - POSTGRES_USER=${DATABASE_USER:-admin} - - POSTGRES_PASSWORD=${DATABASE_PASSWORD:-admin} - - POSTGRES_DB=${DATABASE_DB_NAME:-cogstack} + PGSSLMODE: verify-full + PGSSLROOTCERT: /run/postgresql/tls/ca.crt + POSTGRES_USER: ${DATABASE_USER:-admin} + POSTGRES_PASSWORD: ${DATABASE_PASSWORD:?Set the PostgreSQL bootstrap password} + POSTGRES_DB: ${DATABASE_DB_NAME:-cogstack} + POSTGRES_DB_SCHEMA_PREFIX: ${POSTGRES_DB_SCHEMA_PREFIX:-cogstack_db} + COGSTACK_INGEST_USER: ${DATABASE_INGEST_USER:-cogstack_ingest} + COGSTACK_INGEST_PASSWORD: ${DATABASE_INGEST_PASSWORD:?Set the PostgreSQL ingestion password} volumes: - ../services/cogstack-db/pgsql:/docker-entrypoint-initdb.d:ro # data persistence - databank-vol:/var/lib/postgresql - command: postgres -c "max_connections=${POSTGRES_DB_MAX_CONNECTIONS:-100}" + - type: bind + source: ${POSTGRES_DATABANK_TLS_DIR:-../security/certificates/postgresql/databank} + target: /run/database-tls + read_only: true + bind: + create_host_path: false ports: - - 5558:5432 + - "${POSTGRES_DATABANK_OUTPUT_HOST:-127.0.0.1}:${POSTGRES_DATABANK_OUTPUT_PORT:-5558}:5432" expose: - 5432 networks: @@ -288,21 +354,29 @@ services: image: &mssql-image mcr.microsoft.com/mssql/server:2019-latest@sha256:07d7dedecdbfcf6776035dc5eefb5bfd9aa1804b2f6b99f7dedb397d8a599069 container_name: cogstack-production-databank-db-mssql environment: - - ACCEPT_EULA=y + ACCEPT_EULA: "Y" + MSSQL_SA_PASSWORD: ${MSSQL_SA_PASSWORD:?Set the SQL Server bootstrap password} volumes: # Keep SQL Server files separate from the PostgreSQL databank volume. - databank-mssql-vol:/var/opt/mssql ports: - - 1433:1433 + - "${MSSQL_OUTPUT_HOST:-127.0.0.1}:${MSSQL_OUTPUT_PORT:-1433}:1433" networks: - cognet cogstack-databank-db-mssql-init: image: *mssql-image restart: "no" - env_file: *db-env environment: MSSQL_SERVER: cogstack-databank-db-mssql,1433 + MSSQL_SA_USER: ${MSSQL_SA_USER:-sa} + MSSQL_SA_PASSWORD: ${MSSQL_SA_PASSWORD:?Set the SQL Server bootstrap password} + DATABASE_DB_NAME: ${DATABASE_DB_NAME:-cogstack} + POSTGRES_DB_SCHEMA_PREFIX: ${POSTGRES_DB_SCHEMA_PREFIX:-cogstack_db} + COGSTACK_INGEST_USER: ${MSSQL_INGEST_USER:-cogstack_ingest} + COGSTACK_INGEST_PASSWORD: ${MSSQL_INGEST_PASSWORD:?Set the MSSQL ingestion password} + MSSQL_TRUST_SERVER_CERTIFICATE: ${MSSQL_TRUST_SERVER_CERTIFICATE:-true} + MSSQL_TLS_HOSTNAME: ${MSSQL_TLS_HOSTNAME:-} depends_on: cogstack-databank-db-mssql: condition: service_started @@ -540,14 +614,15 @@ services: # Security certificates, general - ../security/certificates/elastic/${ELASTICSEARCH_VERSION:-opensearch}/elastic-stack-ca.crt.pem:/usr/share/${KIBANA_VERSION:-opensearch-dashboards}/config/root-ca.crt:ro - - ../security/certificates/elastic/${ELASTICSEARCH_VERSION:-opensearch}/elastic-stack-ca.p12:/usr/share/${KIBANA_VERSION:-opensearch-dashboards}/config/root-ca.p12:ro - ../security/certificates/elastic/${ELASTICSEARCH_VERSION:-opensearch}/elasticsearch/${ES_INSTANCE_NAME_1:-elasticsearch-1}/${ES_INSTANCE_NAME_1:-elasticsearch-1}.crt:/usr/share/${KIBANA_VERSION:-opensearch-dashboards}/config/esnode1.crt:ro - ../security/certificates/elastic/${ELASTICSEARCH_VERSION:-opensearch}/elasticsearch/${ES_INSTANCE_NAME_1:-elasticsearch-1}/${ES_INSTANCE_NAME_1:-elasticsearch-1}.key:/usr/share/${KIBANA_VERSION:-opensearch-dashboards}/config/esnode1.key:ro - ../security/certificates/elastic/${ELASTICSEARCH_VERSION:-opensearch}/elasticsearch/${ES_INSTANCE_NAME_1:-elasticsearch-1}/${ES_INSTANCE_NAME_1:-elasticsearch-1}.p12:/usr/share/${KIBANA_VERSION:-opensearch-dashboards}/config/esnode1.p12:ro # OpenSearch uses some default certs - - ../security/certificates:/certificates:ro + - ../security/certificates/elastic/opensearch/elastic-stack-ca.crt.pem:/certificates/elastic/opensearch/elastic-stack-ca.crt.pem:ro + - ../security/certificates/elastic/opensearch/es_kibana_client.pem:/certificates/elastic/opensearch/es_kibana_client.pem:ro + - ../security/certificates/elastic/opensearch/es_kibana_client.key:/certificates/elastic/opensearch/es_kibana_client.key:ro ports: - "${KIBANA_SERVER_OUTPUT_PORT:-5601}:5601" networks: @@ -666,8 +741,9 @@ services: hostname: nifi shm_size: ${NIFI_DOCKER_SHM_SIZE:-"1g"} environment: - - PYTHONPATH=${NIFI_PYTHONPATH:-/opt/nifi/nifi-current/python/framework} - - JVM_OPTS=${NIFI_JVM_OPTS:--XX:+UseG1GC -XX:MaxGCPauseMillis=200 -XX:+ParallelRefProcEnabled -Djava.security.egd=file:/dev/./urandom} + <<: *nifi-db-environment + PYTHONPATH: ${NIFI_PYTHONPATH:-/opt/nifi/nifi-current/python/framework} + JVM_OPTS: ${NIFI_JVM_OPTS:--XX:+UseG1GC -XX:MaxGCPauseMillis=200 -XX:+ParallelRefProcEnabled -Djava.security.egd=file:/dev/./urandom} deploy: resources: limits: @@ -705,7 +781,8 @@ services: volumes: - ../services/nginx/sites-enabled:/etc/nginx/sites-enabled:ro - ../services/nginx/config/nginx.conf.template:/etc/nginx/templates/nginx.conf.template:ro - - ../security/certificates:/certificates:ro + - ../security/certificates/nifi:/certificates/nifi:ro + - ../security/certificates/root/root-ca.pem:/certificates/root/root-ca.pem:ro ports: - "${NIFI_EXTERNAL_PORT_NGINX:-8443}:${NIFI_INTERNAL_PORT_NGINX:-8443}" networks: diff --git a/docs/deploy/air-gapped.md b/docs/deploy/air-gapped.md index cbec1cf7b..f00e25d78 100644 --- a/docs/deploy/air-gapped.md +++ b/docs/deploy/air-gapped.md @@ -231,15 +231,20 @@ Review the files in `deploy/*.env`, especially: Set the final hostname, IP address, and certificate subject alternative names before generating certificates. Replace any example or default credentials in -`security/env/` with deployment-specific values. Then generate only the CA and -NiFi certificate set required by this guide, from the repository root: +`security/env/` with deployment-specific values. Then generate the certificate +files mounted into NiFi, from the repository root: ```bash make -C deploy init-security-nifi ``` -The broader `init-security` target also prepares certificates for Gitea and the -configured search backend. Use it only when those services and their offline +NiFi initialization also prepares the mounted OpenSearch leaf certificates, +public search CA, and PostgreSQL CA files for existing flows. These files are +required even when the database and search services run +elsewhere or are not started. The mounts retain existing certificate paths while +excluding CA signing keys and PostgreSQL private keys. The broader +`init-security` target also prepares certificates for Gitea and the configured +search backend. Use it only when those services and their offline dependencies are included in the deployment bundle. The OpenSearch host setting `vm.max_map_count=262144` is not required for a NiFi-only deployment. diff --git a/docs/deploy/configuration.md b/docs/deploy/configuration.md index a1d5076d5..cd53d5e5d 100644 --- a/docs/deploy/configuration.md +++ b/docs/deploy/configuration.md @@ -46,6 +46,12 @@ The following security files are also loaded where required: See the `env_file` anchors at the top of `deploy/services.yml` for the exact files consumed by each core service. +Database credentials use explicit, per-service environment mappings. The complete +`users_database.env` file is read by the deployment loader but is not injected +into database containers, NiFi, or nginx. NiFi receives restricted ingestion +credentials. See [Docker database security](../security/databases.md) for role +setup, profile upgrades, and the default PostgreSQL TLS configuration. + ## Standalone service files Services launched from their own Compose projects use environment files below @@ -208,6 +214,13 @@ before starting services: make -C deploy init-security ``` +This includes PostgreSQL server bundles and the public CAs mounted into NiFi. +Use `make -C deploy init-security-postgresql` to prepare only the database +bundles. Their paths are configured by `POSTGRES_SAMPLES_TLS_DIR` and +`POSTGRES_DATABANK_TLS_DIR` in `database.env`; see +[Docker database security](../security/databases.md) for trusted certificate +requirements. + Do not reuse development certificates or tracked development credentials in production. See the [security overview](../security/main.md) for production guidance. diff --git a/docs/deploy/deployment.md b/docs/deploy/deployment.md index f5a536065..05a3305fb 100644 --- a/docs/deploy/deployment.md +++ b/docs/deploy/deployment.md @@ -261,9 +261,10 @@ make -C deploy remote-delete-service \ | `make check-nifi-resources` | Check JVM heap sizes against the container memory limit | | `make check-opensearch-settings` | Check search credentials, encryption keys, and image versions | | `make show-env` | Print environment variables (sorted) | -| `make init-security` | Generate missing NiFi, Gitea, and selected search-backend certificates | +| `make init-security` | Generate missing NiFi, PostgreSQL, Gitea, and selected search-backend certificates | | `make init-security-root-ca` | Generate the shared root CA if missing | -| `make init-security-nifi` | Generate the NiFi certificate set if missing | +| `make init-security-nifi` | Prepare NiFi certificates, mounted OpenSearch leaf/public CA files, and PostgreSQL CA files for maintained flows | +| `make init-security-postgresql` | Generate the samples and databank PostgreSQL TLS bundles if missing | | `make init-security-gitea` | Generate the dedicated Gitea leaf certificate if missing | | `make init-security-opensearch` | Generate the OpenSearch certificate set if missing | | `make init-security-elasticsearch` | Generate the native Elasticsearch certificate set if missing | diff --git a/docs/deploy/services.md b/docs/deploy/services.md index be76508cb..51e465d1b 100644 --- a/docs/deploy/services.md +++ b/docs/deploy/services.md @@ -195,6 +195,10 @@ The loader resolves its input paths relative to that directory. Set `POSTGRES_SAMPLES_DB_NAME` in `deploy/database.env` to choose the sample database name; Compose passes it to the official image as `POSTGRES_DB`. +TLS is required by default. Run `make -C deploy init-security-postgresql` (or +`init-security`) before starting the service. The sample JDBC URL verifies the +server certificate and hostname using the CA mounted into NiFi. + ### Access - `localhost:5554` @@ -208,6 +212,10 @@ name; Compose passes it to the official image as `POSTGRES_DB`. Local development credentials are defined in `security/env/users_database.env`. Override them for non-local deployments. +NiFi uses the read-only `POSTGRES_INGEST_USER_SAMPLES` account rather than the +bootstrap administrator. See [Docker database security](../security/databases.md) +for credential configuration, existing-volume upgrades, and TLS. + --- ## 🏦 Cogstack databank production DB (Production only: PgSQL, MSSQL) @@ -226,6 +234,9 @@ Where `` can be `mssql` or `pgsql`. PostgreSQL applies its initialization SQL only when its data volume is first created. Compose sets `POSTGRES_DB` from `DATABASE_DB_NAME`, and the official PostgreSQL entrypoint creates the database before running the bundled loader. +PostgreSQL requires TLS by default; generate its certificate bundle with +`make -C deploy init-security-postgresql` (also included in `init-security`) or +provide a trusted bundle through `POSTGRES_DATABANK_TLS_DIR`. SQL Server runs as the main process in its own container; a separate `cogstack-databank-db-mssql-init` client job waits for a successful query, creates the configured database if needed, applies `cogstack_db*.sql` files in filename @@ -248,9 +259,9 @@ database configuration. Its readiness wait defaults to 60 attempts, two seconds apart (each connection attempt has a five-second login timeout). Override `MSSQL_INIT_RETRIES` and `MSSQL_INIT_RETRY_DELAY_SECONDS` in the initializer's environment if necessary. The bundled tables use the `CogStack` SQL schema -within the selected database. The job accepts the reference server's self-signed -TLS certificate; adapt client trust settings when supplying a managed server -certificate. +within the selected database. The job always requests TLS. The development +default `MSSQL_TRUST_SERVER_CERTIFICATE=true` accepts the self-signed server +certificate; set `false` with a trusted CA to enable certificate verification. Bundled MSSQL initialization preserves existing tables and data on repeat runs. It does not alter existing column definitions. Apply explicit migrations when @@ -285,6 +296,12 @@ volume before switching. No existing volumes are automatically moved or deleted. PostgreSQL and MSSQL development credentials are defined in `security/env/users_database.env`. Production credentials must be injected through deployment-local configuration or a secret manager. +Applications use the separate `DATABASE_INGEST_USER` and `MSSQL_INGEST_USER` +accounts. Their grants cover read/write access to application tables, without +database or schema administration. Host ports default to loopback. See +[Docker database security](../security/databases.md) for scope, provisioning, +network overrides, and verified TLS deployment. + ### Access - PgSQL: `localhost:5558` → container `5432` diff --git a/docs/deploy/workflows/database_ocr_to_opensearch.md b/docs/deploy/workflows/database_ocr_to_opensearch.md index 39543c425..6d126b618 100644 --- a/docs/deploy/workflows/database_ocr_to_opensearch.md +++ b/docs/deploy/workflows/database_ocr_to_opensearch.md @@ -59,7 +59,8 @@ OpenSearch indexing. binary document column. - NiFi can resolve `ocr-service` on the Docker network, or the `InvokeHTTP` URL is changed to a reachable endpoint. -- Database, OpenSearch, keystore, and truststore passwords are entered after import. +- Database credentials use ingestion environment references. Enter the OpenSearch, + keystore, and truststore passwords after import. ## 📥 Import the flow @@ -82,18 +83,18 @@ OpenSearch indexing. | Property | Bundled value | |---|---| -| Database Connection URL | `jdbc:postgresql://samples-db:5432/db_samples` | +| Database Connection URL | `${POSTGRES_SAMPLES_JDBC_URL}` | | Database Driver Class Name | `org.postgresql.Driver` | | Database Driver Locations | `/opt/nifi/drivers/postgresql-42.7.7.jar` | -| Database User | Value of `POSTGRES_USER_SAMPLES`, `test` by default | -| Password | Value of `POSTGRES_PASSWORD_SAMPLES` | +| Database User | `${POSTGRES_INGEST_USER_SAMPLES}`, `cogstack_samples` by default | +| Password | `${POSTGRES_INGEST_PASSWORD_SAMPLES}` | | Password Source | `PASSWORD` | | Max Total Connections | `8` | | Maximum Idle Connections | `8` | | Minimum Idle Connections | `0` | | Max Wait Time | `500 millis` | -Read the credentials from `security/env/users_database.env`. When using another source, +The bundled database credentials reference ingestion environment variables. When using another source, change the URL, driver class, driver JAR, and credentials together. Apply and enable the service. diff --git a/docs/deploy/workflows/database_to_opensearch.md b/docs/deploy/workflows/database_to_opensearch.md index 26fda846e..196bbf4e5 100644 --- a/docs/deploy/workflows/database_to_opensearch.md +++ b/docs/deploy/workflows/database_to_opensearch.md @@ -31,7 +31,7 @@ Failure and retry relationships are routed to funnels for inspection. | Setting | Value | |---|---| | Database URL | `jdbc:postgresql://samples-db:5432/db_samples` | -| Database user | `test` | +| Database user | `cogstack_samples` | | JDBC driver | `/opt/nifi/drivers/postgresql-42.7.7.jar` | | Source table | `medical_reports_text` | | Incremental/partition column | `docid` | @@ -82,19 +82,19 @@ Configure the sample PostgreSQL connection as follows: | Property | Bundled value | |---|---| -| Database Connection URL | `jdbc:postgresql://samples-db:5432/db_samples` | +| Database Connection URL | `${POSTGRES_SAMPLES_JDBC_URL}` | | Database Driver Class Name | `org.postgresql.Driver` | | Database Driver Locations | `/opt/nifi/drivers/postgresql-42.7.7.jar` | -| Database User | Value of `POSTGRES_USER_SAMPLES`, `test` by default | -| Password | Value of `POSTGRES_PASSWORD_SAMPLES` | +| Database User | `${POSTGRES_INGEST_USER_SAMPLES}`, `cogstack_samples` by default | +| Password | `${POSTGRES_INGEST_PASSWORD_SAMPLES}` | | Password Source | `PASSWORD` | | Max Total Connections | `8` | | Maximum Idle Connections | `8` | | Minimum Idle Connections | `0` | | Max Wait Time | `500 millis` | -Read the sensitive value from `security/env/users_database.env`; do not store it in the -flow JSON. If using another database, update the URL, driver class, driver JAR, and +The bundled database properties reference ingestion environment variables. Keep +password values out of flow JSON; configure them through the deployment profile. If using another database, update the URL, driver class, driver JAR, and credentials together. Apply the configuration and enable the service. A driver error usually means the JAR path does not exist inside the NiFi container. diff --git a/docs/security/certificates.md b/docs/security/certificates.md index d2981741b..5750acc6e 100644 --- a/docs/security/certificates.md +++ b/docs/security/certificates.md @@ -4,7 +4,7 @@ This section describes the full structure of the `security/certificates/` direct The supported entry point is `make -C deploy init-security`. It generates the shared **Root Certificate Authority (CA)** and the required service certificates locally. Generated certificate material is ignored by Git. -The shared root CA signs the locally generated NiFi, OpenSearch, and Gitea certificates. Native Elasticsearch is the exception: it uses Elastic's certificate tooling to generate a backend-specific CA and certificate set. +The shared root CA signs the locally generated NiFi, PostgreSQL, OpenSearch, and Gitea certificates. Native Elasticsearch is the exception: it uses Elastic's certificate tooling to generate a backend-specific CA and certificate set. --- @@ -38,8 +38,7 @@ security/ │ │ │ ├── elasticsearch-*-truststore.key # Truststores │ │ │ └── http-elasticsearch-*.csr/key # HTTP layer certs │ ├── es_kibana_client.{pem,key,p12,csr} # Kibana client certs - │ ├── elastic-stack-ca.* # OpenSearch cluster CA - │ └── elastic-stack-ca.* # Shared CA copied for OpenSearch consumers + │ └── elastic-stack-ca.crt.pem # Public shared CA certificate only │ ├── nifi/ # NiFi HTTPS and toolkit certificates │ ├── nifi.{crt,key,p12,pem,csr} # Primary NiFi node certificates @@ -49,6 +48,10 @@ security/ ├── gitea/ # Dedicated Gitea leaf certificate │ └── gitea.{crt,key,pem,csr} │ + ├── postgresql/ # PostgreSQL server TLS bundles + │ └── {samples,databank}/ + │ └── server.crt, server.key, ca.crt + │ └── root/ # Root Certificate Authority (CA) ├── root-ca.key, root-ca.pem # Private key and public cert ├── root-ca.p12, root-ca-keystore.jks # PKCS#12 and Java Keystore formats @@ -56,6 +59,10 @@ security/ └── root-ca.csr, root-ca.srl # Certificate signing request and serial ``` +OpenSearch generators copy only the public CA certificate into the OpenSearch +directory. The shared CA signing key and key-bearing CA stores remain under +`root/` and are not mounted into service containers. + --- ### ⚙️ Environment configuration @@ -79,8 +86,9 @@ The Make targets load the required `.env` files automatically. The lower-level s ### 🛠️ Generation workflow -From the repository root, initialize all certificates required by NiFi, Gitea, -and the search backend selected by `ELASTICSEARCH_VERSION` in `deploy/elasticsearch.env`: +From the repository root, initialize all certificates required by NiFi, +PostgreSQL, Gitea, and the search backend selected by `ELASTICSEARCH_VERSION` in +`deploy/elasticsearch.env`: ```bash make -C deploy init-security @@ -91,11 +99,16 @@ You can also initialize certificate sets independently: ```bash make -C deploy init-security-root-ca make -C deploy init-security-nifi +make -C deploy init-security-postgresql make -C deploy init-security-gitea make -C deploy init-security-opensearch make -C deploy init-security-elasticsearch ``` +`init-security-nifi` also prepares the OpenSearch leaf certificates and public +CA, plus the PostgreSQL CA files mounted at the paths used by maintained flows, +regardless of the selected search backend or whether the databases are started. + The initialization targets are idempotent: complete certificate sets are left unchanged. Root CA and native Elasticsearch generation refuse to overwrite an existing incomplete directory automatically, preventing accidental CA rotation. diff --git a/docs/security/databases.md b/docs/security/databases.md new file mode 100644 index 000000000..339595ded --- /dev/null +++ b/docs/security/databases.md @@ -0,0 +1,145 @@ +# Docker database security + +## Accounts and credential delivery + +The tracked `security/env/users_database.env` contains public development defaults. +For shared or production deployments, copy it into the selected +`COGSTACK_CONFIG_DIR/security/` directory and replace the passwords there. +Profiles replace whole files, so existing profiles must add all six new ingestion +username/password settings. + +| Service | Bootstrap account settings | Application account settings | Application privileges | +|---|---|---|---| +| Samples PostgreSQL | `POSTGRES_USER_SAMPLES`, `POSTGRES_PASSWORD_SAMPLES` | `POSTGRES_INGEST_USER_SAMPLES`, `POSTGRES_INGEST_PASSWORD_SAMPLES` | SELECT on `public` tables/views | +| Databank PostgreSQL | `DATABASE_USER`, `DATABASE_PASSWORD` | `DATABASE_INGEST_USER`, `DATABASE_INGEST_PASSWORD` | SELECT, INSERT, UPDATE, DELETE on `public` tables and sequence access | +| Databank MSSQL | `MSSQL_SA_USER`, `MSSQL_SA_PASSWORD` | `MSSQL_INGEST_USER`, `MSSQL_INGEST_PASSWORD` | SELECT, INSERT, UPDATE, DELETE on the `CogStack` schema | + +Database services use explicit environment mappings rather than loading the +complete database credentials file. PostgreSQL needs its bootstrap and ingestion +credentials to initialize roles; the MSSQL server gets only its SA password, and +the separate initializer gets that password plus its ingestion credentials. +NiFi gets ingestion credentials only. Its certificate mounts cover NiFi/search +material and public root/database CAs. Database private keys and the root signing +key are excluded from NiFi, nginx, and Dashboards mounts. + +Application roles cannot administer databases or create application schemas. +PostgreSQL grants include existing objects and future objects created by the +bootstrap owner in `public`. Site SQL using other schemas or migration owners +must supply explicit grants. Access SQL remains mounted and editable. + +Provisioning rejects an ingestion name that reuses an administrator, object +owner, or role member. PostgreSQL also rejects inherited CREATE permissions from +legacy PUBLIC grants instead of reporting that the application role is restricted. + +Environment scoping is not an encrypted secret store. Protect deployment-local +configuration and Docker daemon access, or inject values using deployment +tooling. Avoid sharing `make show-env` or resolved Compose output containing +credentials. + +## Existing volumes and flows + +Fresh PostgreSQL initialization configures ingestion access after loading its +schemas/data. These Make targets also apply access SQL against existing volumes: + +```bash +make -C deploy start-samples +make -C deploy start-production-db +make -C deploy start-production-db-mssql +``` + +They recreate service configuration when needed and provision the named +application accounts. They do not reset data or demote the existing bootstrap +administrator. Bootstrap credentials must match the existing database; changing +the PostgreSQL administrator password in an environment file does not rotate the +stored password. + +The MSSQL target applies the schema and then configures the application login. +The bundled schema preserves existing tables; column upgrades require migrations. + +Existing NiFi flows must switch their database controller services to ingestion +credentials. The maintained database/OCR templates use: + +- URL: `${POSTGRES_SAMPLES_JDBC_URL}` +- User: `${POSTGRES_INGEST_USER_SAMPLES}` +- Password: `${POSTGRES_INGEST_PASSWORD_SAMPLES}` + +These are environment references, not password values committed in the flow. +Existing running flows are not overwritten. For databank pools, use the +corresponding ingestion settings from the table above. + +## Host access + +Database host ports default to `127.0.0.1`: + +| Settings | Default | +|---|---| +| `POSTGRES_SAMPLES_OUTPUT_HOST`, `POSTGRES_SAMPLES_OUTPUT_PORT` | `127.0.0.1:5554` | +| `POSTGRES_DATABANK_OUTPUT_HOST`, `POSTGRES_DATABANK_OUTPUT_PORT` | `127.0.0.1:5558` | +| `MSSQL_OUTPUT_HOST`, `MSSQL_OUTPUT_PORT` | `127.0.0.1:1433` | + +Override these in the deployment's `database.env` when remote host access is +required. Container clients keep using Compose service DNS and internal ports. + +## PostgreSQL TLS + +Both PostgreSQL services require TLS in `deploy/services.yml`. Bundled clients +use certificate and hostname verification; plaintext network connections are +rejected. No additional Compose file is needed. + +Generate the local reference bundles before starting services: + +```bash +make -C deploy init-security-postgresql +``` + +This target is also included in `make -C deploy init-security`. It signs the +database certificates with the deployment's shared root CA and leaves complete +existing bundles unchanged. The default directories, configured in +`deploy/database.env`, are: + +| Setting | Default path, relative to `deploy/` | +|---|---| +| `POSTGRES_SAMPLES_TLS_DIR` | `../security/certificates/postgresql/samples` | +| `POSTGRES_DATABANK_TLS_DIR` | `../security/certificates/postgresql/databank` | + +Profiles can override either TLS directory; omitted settings use these defaults. + +For production, override these settings with directories containing trusted +`server.crt`, `server.key`, and `ca.crt` files. The server certificates must have +subject alternative names covering `samples-db` and `cogstack-databank-db`, +respectively, plus any other hostname used by clients. Keep host private keys +restricted. At startup, each service copies its bundle into a PostgreSQL-owned +runtime directory with the required permissions, so host key ownership does not +need to match the container's PostgreSQL UID. Certificate files remain +deployment-local inputs. + +The services enable TLS 1.2 or later and SCRAM password authentication while +retaining local Unix-socket provisioning. The Make start targets apply access +configuration against existing volumes using verified TLS. NiFi receives only +the public database CAs at `/run/database-cas/samples-ca.crt` and +`/run/database-cas/databank-ca.crt`. The sample JDBC URL defaults to +`sslmode=verify-full`. For a databank controller service, use: + +```text +jdbc:postgresql://cogstack-databank-db:5432/cogstack?sslmode=verify-full&sslrootcert=/run/database-cas/databank-ca.crt +``` + +Adjust the database name if necessary. Other clients must also configure CA and +hostname verification; server encryption alone does not establish server identity. + +## MSSQL TLS + +The initializer always passes `-N` to request encryption. +`MSSQL_TRUST_SERVER_CERTIFICATE=true` is the explicit local-development default +for the reference server's self-signed certificate. + +For managed certificates, configure the SQL Server certificate/key and +server-side encryption policy, install the issuing CA in the initializer image's +OS trust store, and set `MSSQL_TRUST_SERVER_CERTIFICATE=false`. +An optional `MSSQL_TLS_HOSTNAME` supplies the certificate hostname via `-F`. +Verified mode never falls back to `-C`. + +A custom initializer image can be supplied through a deployment-specific Compose +override and must retain an immutable digest. The hostname setting does not +install a CA or a server certificate. Other MSSQL clients, including NiFi JDBC +pools, must require encryption and certificate verification independently. diff --git a/docs/security/elasticsearch_opensearch.md b/docs/security/elasticsearch_opensearch.md index 59d31e1be..6fadcafdc 100644 --- a/docs/security/elasticsearch_opensearch.md +++ b/docs/security/elasticsearch_opensearch.md @@ -121,7 +121,8 @@ security/certificates/elastic/ │ ├── sample-kibana.yml │ └── README.txt └── opensearch/ - ├── admin.*, es_kibana_client.*, elastic-stack-ca.* + ├── admin.*, es_kibana_client.* + ├── elastic-stack-ca.crt.pem # Public CA certificate only └── elasticsearch/{1,2,3}/... ``` @@ -170,6 +171,7 @@ This produces: |------|----------| | `admin.crt`, `admin.key.pem` | Administrative client certificate for security management | | `es_kibana_client.pem`, `es_kibana_client.key` | HTTPS server certificate for OpenSearch Dashboards | +| `elastic-stack-ca.crt.pem` | Public shared CA certificate; signing keys stay under `root/` | | `*.jks` | Node keystores/truststores for HTTPS and inter-node encryption | The resulting certificates are placed in: diff --git a/docs/security/main.md b/docs/security/main.md index 1b08b5d88..2b2f43344 100644 --- a/docs/security/main.md +++ b/docs/security/main.md @@ -17,11 +17,12 @@ Security is achieved through: Generated certificates and private keys are deployment-local and ignored by Git. Run `make -C deploy init-security` before starting services. The tracked `security/env/*.env` files contain public development defaults only; never use those values as production secrets. -## 🧩 Components secured with HTTPS +## 🧩 Components secured with TLS | Service | Certificate location | Supported initialization target | |---------|----------------------|---------------------------------| | NiFi | `security/certificates/nifi/` | `make -C deploy init-security-nifi` | +| PostgreSQL samples and databank | `security/certificates/postgresql/` | `make -C deploy init-security-postgresql` | | Elasticsearch | `security/certificates/elastic/elasticsearch/` | `make -C deploy init-security-elasticsearch` | | OpenSearch and Dashboards | `security/certificates/elastic/opensearch/` | `make -C deploy init-security-opensearch` | | Gitea | `security/certificates/gitea/` | `make -C deploy init-security-gitea` | @@ -41,6 +42,7 @@ security/ │ ├── elastic/ # Elasticsearch / OpenSearch + Kibana certs │ ├── nifi/ # Apache NiFi leaf certificate and Java stores │ ├── gitea/ # Gitea leaf certificate +│ ├── postgresql/ # Samples and databank PostgreSQL TLS bundles │ └── root/ # Root CA files and truststores │ ├── env/ # Environment variable definitions for certs and users diff --git a/docs/security/nifi.md b/docs/security/nifi.md index 7ed1fda23..a6b84fc9a 100644 --- a/docs/security/nifi.md +++ b/docs/security/nifi.md @@ -35,9 +35,14 @@ For securing Apache NiFi endpoints with certificates, see [the official document Before starting the NiFi container: -- run `make -C deploy init-security-nifi` from the repository root. This creates the shared root CA when needed, then generates the NiFi certificate bundle without replacing a complete existing bundle. +- run `make -C deploy init-security-nifi` from the repository root. This creates the shared root CA when needed, then prepares the NiFi bundle, mounted OpenSearch leaf certificates and public CA, and PostgreSQL CA files used by maintained flows. Complete existing certificate sets are preserved, including when a different search backend is selected. For supplied PostgreSQL bundles, see [Docker database security](./databases.md). - set `nifi.sensitive.props.key` to a stable value (minimum 12 characters). +These certificate files are required even when the database and search services +run elsewhere or are not started. The combined `init-security` target covers all +these sets. NiFi mounts only its required leaf certificates and public +CAs; CA signing keys and PostgreSQL private keys are excluded. + Example (`nifi/conf/nifi.properties`): ```properties diff --git a/mkdocs.yml b/mkdocs.yml index b3a6c54af..8b765eaea 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -67,6 +67,7 @@ nav: - security/main.md - 🏛️ Certificates and Root CA: security/certificates.md - 🗄️ Additional services: security/services.md + - 🗃️ Docker database security: security/databases.md - 🔐 NiFi TLS & Admin Access Setup (with NGINX): security/nifi.md - 🌐 Elasticsearch / OpenSearch Security: security/elasticsearch_opensearch.md diff --git a/nifi/user_scripts/dto/database_config.py b/nifi/user_scripts/dto/database_config.py index 0999ebb88..8cf1bbffc 100644 --- a/nifi/user_scripts/dto/database_config.py +++ b/nifi/user_scripts/dto/database_config.py @@ -19,10 +19,12 @@ class DatabaseConfig(BaseSettings): host: str = Field(default="localhost", validation_alias=AliasChoices("POSTGRES_HOST")) port: int = Field(default=5432,validation_alias=AliasChoices("POSTGRES_PORT"), ge=1, le=65535) - database_name : str = Field(default="db_samples", validation_alias=AliasChoices("DB", "DB_NAME")) - username: str = Field(default="test", validation_alias=AliasChoices("POSTGRES_USER_SAMPLES", "POSTGRES_USER")) - password: SecretStr = Field(default_factory=lambda: SecretStr("test"), - validation_alias=AliasChoices("POSTGRES_PASSWORD_SAMPLES", + database_name : str = Field(default="db_samples", validation_alias=AliasChoices( + "POSTGRES_SAMPLES_DB_NAME", "DB", "DB_NAME")) + username: str = Field(default="cogstack_samples", validation_alias=AliasChoices( + "POSTGRES_INGEST_USER_SAMPLES", "POSTGRES_USER_SAMPLES", "POSTGRES_USER")) + password: SecretStr = Field(validation_alias=AliasChoices("POSTGRES_INGEST_PASSWORD_SAMPLES", + "POSTGRES_PASSWORD_SAMPLES", "password", "POSTGRES_PASSWORD")) timeout: PositiveInt = Field(default=60, validation_alias=AliasChoices("TIMEOUT")) diff --git a/scripts/tests/smoke_postgres_services.sh b/scripts/tests/smoke_postgres_services.sh index f433d0184..d4f13bcc7 100755 --- a/scripts/tests/smoke_postgres_services.sh +++ b/scripts/tests/smoke_postgres_services.sh @@ -17,9 +17,10 @@ if ! command -v docker >/dev/null 2>&1; then fi CONTAINER_NAME="${POSTGRES_SMOKE_CONTAINER_NAME:-cogstack-samples-db}" -DB_USER="${POSTGRES_USER_SAMPLES:-test}" -DB_PASSWORD="${POSTGRES_PASSWORD_SAMPLES:-test}" +DB_USER="${POSTGRES_INGEST_USER_SAMPLES:-cogstack_samples}" +DB_PASSWORD="${POSTGRES_INGEST_PASSWORD_SAMPLES:?Set the sample ingestion password}" DB_NAME="${POSTGRES_SAMPLES_DB_NAME:-db_samples}" +DB_HOST="${POSTGRES_SMOKE_HOST:-samples-db}" START_SERVICES="${POSTGRES_SMOKE_START_SERVICES:-1}" RETRIES="${POSTGRES_SMOKE_RETRIES:-30}" @@ -42,7 +43,7 @@ check_pg_isready() { if docker exec \ -e PGPASSWORD="$DB_PASSWORD" \ "$CONTAINER_NAME" \ - pg_isready -U "$DB_USER" -d "$DB_NAME" >/dev/null 2>&1; then + pg_isready -h "$DB_HOST" -U "$DB_USER" -d "$DB_NAME" >/dev/null 2>&1; then echo "OK: pg_isready for database ${DB_NAME}" return 0 fi @@ -51,14 +52,15 @@ check_pg_isready() { return 1 } -check_basic_query() { +check_tls_query() { local result if ! result="$( docker exec \ -e PGPASSWORD="$DB_PASSWORD" \ "$CONTAINER_NAME" \ - psql -U "$DB_USER" -d "$DB_NAME" -tAc 'SELECT 1' + psql -h "$DB_HOST" -U "$DB_USER" -d "$DB_NAME" -tAc \ + 'SELECT 1 FROM pg_stat_ssl WHERE pid = pg_backend_pid() AND ssl' )"; then echo "FAIL: query check failed for database ${DB_NAME}" return 1 @@ -66,18 +68,18 @@ check_basic_query() { result="$(echo "$result" | tr -d '[:space:]')" if [[ "$result" == "1" ]]; then - echo "OK: query check returned 1" + echo "OK: query used a TLS connection" return 0 fi - echo "FAIL: query check returned unexpected value: ${result}" + echo "FAIL: TLS query check returned unexpected value: ${result}" return 1 } run_checks() { check_container_running && check_pg_isready && - check_basic_query + check_tls_query } if [[ "$START_SERVICES" != "0" ]]; then @@ -86,6 +88,8 @@ if [[ "$START_SERVICES" != "0" ]]; then exit 1 fi + echo "Preparing PostgreSQL certificates using 'make -C deploy init-security-postgresql'." + make -C "${ROOT_DIR}/deploy" init-security-postgresql echo "Starting samples Postgres service using 'make -C deploy start-samples'." make -C "${ROOT_DIR}/deploy" start-samples fi diff --git a/scripts/tests/test_database_deployment_security.py b/scripts/tests/test_database_deployment_security.py new file mode 100755 index 000000000..a4d5ff164 --- /dev/null +++ b/scripts/tests/test_database_deployment_security.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 +"""Check database credential isolation, host publishing, and default TLS.""" + +import json +import os +import subprocess +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] + + +def render(*files, extra_env=None): + command = 'source "$1" >/dev/null && shift && docker compose "$@" config --format json' + args = ["bash", "-c", command, "database-security", str(ROOT / "deploy/export_env_vars.sh")] + for file in files: + args.extend(["-f", str(ROOT / "deploy" / file)]) + result = subprocess.run( + args, cwd=ROOT, env={**os.environ, **(extra_env or {})}, + capture_output=True, text=True, timeout=30, check=True, + ) + return json.loads(result.stdout) + + +class DatabaseDeploymentSecurityTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.config = render("services.yml") + + def test_database_containers_receive_only_their_own_credentials(self): + services = self.config["services"] + for name in ("samples-db", "cogstack-databank-db"): + env = services[name]["environment"] + self.assertIn("POSTGRES_PASSWORD", env) + self.assertIn("COGSTACK_INGEST_PASSWORD", env) + self.assertNotIn("MSSQL_SA_PASSWORD", env) + self.assertNotIn("CLICKHOUSE_PASSWORD", env) + self.assertNotIn("DATABASE_PASSWORD", env) + self.assertNotIn("POSTGRES_PASSWORD", services["cogstack-databank-db-mssql"]["environment"]) + self.assertNotIn("COGSTACK_INGEST_PASSWORD", services["cogstack-databank-db-mssql"]["environment"]) + env = services["cogstack-databank-db-mssql-init"]["environment"] + self.assertIn("MSSQL_SA_PASSWORD", env) + self.assertIn("COGSTACK_INGEST_PASSWORD", env) + self.assertNotIn("POSTGRES_PASSWORD", env) + + def test_nifi_has_ingestion_credentials_without_database_admin_files(self): + forbidden = {"DATABASE_USER", "DATABASE_PASSWORD", "POSTGRES_USER_SAMPLES", + "POSTGRES_PASSWORD_SAMPLES", "MSSQL_SA_USER", "MSSQL_SA_PASSWORD", "CLICKHOUSE_PASSWORD"} + for file in ("services.yml", "services.dev.yml"): + services = render(file)["services"] + nifi = services["nifi"] + self.assertTrue(forbidden.isdisjoint(nifi["environment"])) + self.assertIn("POSTGRES_INGEST_PASSWORD_SAMPLES", nifi["environment"]) + self.assertIn("DATABASE_INGEST_PASSWORD", nifi["environment"]) + self.assertIn("MSSQL_INGEST_PASSWORD", nifi["environment"]) + self.assertNotIn("/security", [v["target"] for v in nifi["volumes"]]) + self.assertTrue(forbidden.isdisjoint(services["nifi-nginx"]["environment"])) + url = nifi["environment"]["POSTGRES_SAMPLES_JDBC_URL"] + self.assertIn("sslmode=verify-full", url) + self.assertIn("sslrootcert=/run/database-cas/samples-ca.crt", url) + for name in ("samples", "databank"): + mount = next(v for v in nifi["volumes"] if v["target"] == f"/run/database-cas/{name}-ca.crt") + self.assertEqual(Path(mount["source"]).name, "ca.crt") + self.assertTrue(mount["read_only"]) + self.assertFalse(mount["bind"]["create_host_path"]) + # Neither the database keys nor the root signing key belong in client containers. + for service_name in ("nifi", "nifi-nginx", "kibana"): + if service_name not in services: + continue + for mount in services[service_name]["volumes"]: + if mount["type"] != "bind": + continue + source = Path(mount["source"]).resolve() + for private_key in ("root/root-ca.key", "root/root-ca.p12", "root/root-ca-keystore.jks", + "elastic/opensearch/elastic-stack-ca.key.pem", "elastic/opensearch/elastic-stack-ca.p12", + "elastic/elasticsearch/elastic-stack-ca.key.pem", "elastic/elasticsearch/elastic-stack-ca.p12", + "postgresql/samples/server.key", "postgresql/databank/server.key"): + self.assertFalse((ROOT / "security/certificates" / private_key).is_relative_to(source), + f"{service_name} exposes {private_key}") + + def test_database_ports_are_published_on_loopback(self): + for name in ("samples-db", "cogstack-databank-db", "cogstack-databank-db-mssql"): + with self.subTest(service=name): + for port in self.config["services"][name]["ports"]: + self.assertEqual(port["host_ip"], "127.0.0.1") + + def test_database_bootstrap_mounts_remain_read_only(self): + services = self.config["services"] + for name in ("samples-db", "cogstack-databank-db", "cogstack-databank-db-mssql-init"): + target = "/bootstrap" if name.endswith("-init") else "/docker-entrypoint-initdb.d" + mount = next(v for v in services[name]["volumes"] if v["target"] == target) + self.assertTrue(mount["read_only"]) + + def test_default_postgres_requires_tls_and_stages_keys_with_private_permissions(self): + for name in ("samples-db", "cogstack-databank-db"): + service = self.config["services"][name] + self.assertEqual(service["environment"]["PGSSLMODE"], "verify-full") + self.assertEqual(service["environment"]["PGSSLROOTCERT"], "/run/postgresql/tls/ca.crt") + self.assertIn("ssl=on", service["command"]) + self.assertIn("ssl_min_protocol_version=TLSv1.2", service["command"]) + self.assertIn("ssl_key_file=/run/postgresql/tls/server.key", service["command"]) + self.assertIn("hba_file=/docker-entrypoint-initdb.d/access/pg_hba_tls.conf", service["command"]) + script = service["entrypoint"][2] + self.assertIn("install -m 600 -o postgres -g postgres /run/database-tls/server.key", script) + self.assertIn('exec /usr/local/bin/docker-entrypoint.sh "$$@"', script) + mount = next(v for v in service["volumes"] if v["target"] == "/run/database-tls") + self.assertTrue(mount["read_only"]) + self.assertFalse(mount["bind"]["create_host_path"]) + for service_path in ("pgsamples", "cogstack-db/pgsql"): + hba = (ROOT / "services" / service_path / "access/pg_hba_tls.conf").read_text() + self.assertIn("hostnossl all all all reject", hba) + self.assertIn("hostssl all all all scram-sha-256", hba) + + def test_certificate_directories_are_configurable_for_servers_and_clients(self): + with tempfile.TemporaryDirectory(prefix="cogstack tls model ") as temp: + c = render("services.yml", extra_env={ + "POSTGRES_SAMPLES_TLS_DIR": temp, "POSTGRES_DATABANK_TLS_DIR": temp, + }) + for name in ("samples-db", "cogstack-databank-db"): + service = c["services"][name] + mount = next(v for v in service["volumes"] if v["target"] == "/run/database-tls") + self.assertEqual(mount["source"], temp) + for name in ("samples", "databank"): + mount = next(v for v in c["services"]["nifi"]["volumes"] if v["target"] == f"/run/database-cas/{name}-ca.crt") + self.assertEqual(mount["source"], str(Path(temp) / "ca.crt")) + + def test_older_database_profiles_get_tls_defaults(self): + with tempfile.TemporaryDirectory(prefix="cogstack old profile ") as temp: + settings = (ROOT / "deploy/database.env").read_text().splitlines() + omitted = ("POSTGRES_SAMPLES_TLS_DIR=", "POSTGRES_DATABANK_TLS_DIR=", "POSTGRES_SAMPLES_JDBC_URL=") + settings = [line for line in settings if not line.startswith(omitted)] + settings = [line.replace("POSTGRES_SAMPLES_DB_NAME=db_samples", "POSTGRES_SAMPLES_DB_NAME=profile_samples") for line in settings] + (Path(temp) / "database.env").write_text("\n".join(settings) + "\n") + c = render("services.yml", extra_env={"COGSTACK_CONFIG_DIR": temp}) + url = c["services"]["nifi"]["environment"]["POSTGRES_SAMPLES_JDBC_URL"] + self.assertIn("/profile_samples?sslmode=verify-full", url) + for name, directory in (("samples-db", "samples"), ("cogstack-databank-db", "databank")): + mount = next(v for v in c["services"][name]["volumes"] if v["target"] == "/run/database-tls") + self.assertEqual(mount["source"], str(ROOT / "security/certificates/postgresql" / directory)) + + def test_reference_flows_use_ingestion_environment_references(self): + for name in ("opensearch_ingest_docs_db_to_es.json", "opensearch_ingest_docs_db_ocr_service_to_es.json"): + d = json.loads((ROOT / "nifi/user_templates" / name).read_text()) + pool = next(v for v in d["flowContents"]["controllerServices"] if v["name"] == "DBCPConnectionPool") + self.assertEqual(pool["properties"]["Database User"], "${POSTGRES_INGEST_USER_SAMPLES}") + self.assertEqual(pool["properties"]["Password"], "${POSTGRES_INGEST_PASSWORD_SAMPLES}") + self.assertEqual(pool["properties"]["Database Connection URL"], "${POSTGRES_SAMPLES_JDBC_URL}") + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tests/test_mssql_initialization.py b/scripts/tests/test_mssql_initialization.py index 9044520d3..e211d320a 100755 --- a/scripts/tests/test_mssql_initialization.py +++ b/scripts/tests/test_mssql_initialization.py @@ -42,6 +42,7 @@ def setUp(self): stream.write(json.dumps({ "args": args, "password_from_env": os.environ.get("SQLCMDPASSWORD") == "fixture-password", + "stdin": sys.stdin.read() if "-x" in args else "", }) + "\\n") if "-Q" in args and "SELECT 1" in args[args.index("-Q") + 1]: attempts = sum("-Q" in c["args"] and "SELECT 1" in c["args"][c["args"].index("-Q") + 1] for c in calls) @@ -51,6 +52,8 @@ def setUp(self): sys.exit(7) if "-i" in args and os.environ.get("FAKE_FAIL_FILE") == Path(args[args.index("-i") + 1]).name: sys.exit(7) +if "-x" in args and os.environ.get("FAKE_ACCESS_FAILURE"): + sys.exit(7) """ ) self.sqlcmd.chmod(0o755) @@ -63,6 +66,8 @@ def setUp(self): "MSSQL_INIT_RETRIES": "3", "MSSQL_INIT_RETRY_DELAY_SECONDS": "0", "SQLCMD_BIN": str(self.sqlcmd), + "COGSTACK_INGEST_USER": "fixture_ingest", + "COGSTACK_INGEST_PASSWORD": "fixture-ingest-password", "FAKE_SQLCMD_LOG": str(self.log), } @@ -89,7 +94,7 @@ def test_waits_until_ready_then_applies_sql_in_order_and_exits(self): result = self.run_job(FAKE_NOT_READY="2") self.assertEqual(result.returncode, 0, result.stderr) calls = self.calls() - self.assertEqual(len(calls), 7) + self.assertEqual(len(calls), 8) self.assertEqual( self.applied_files(), ["cogstack_db_01 first.sql", "cogstack_db_02 second.sql", self.annotations.name], @@ -149,6 +154,7 @@ def test_missing_annotation_schema_fails_before_connecting(self): def test_schema_directory_defaults_to_the_mounted_bootstrap_bundle(self): script = self.directory / "init_db.sh" shutil.copy2(INITIALIZER, script) + shutil.copytree(INITIALIZER.parent / "access", self.directory / "access") shutil.copytree(self.schemas, self.directory / "schemas") result = self.run_job(script=script, MSSQL_SCHEMA_DIR="") self.assertEqual(result.returncode, 0, result.stderr) @@ -157,6 +163,33 @@ def test_schema_directory_defaults_to_the_mounted_bootstrap_bundle(self): path = Path(call["args"][call["args"].index("-i") + 1]) self.assertEqual(path.parent, self.directory / "schemas") + def test_verified_tls_requires_encryption_without_trust_bypass(self): + result = self.run_job(MSSQL_TRUST_SERVER_CERTIFICATE="false", MSSQL_TLS_HOSTNAME="db.example.org") + self.assertEqual(result.returncode, 0, result.stderr) + for call in self.calls(): + self.assertIn("-N", call["args"]) + self.assertNotIn("-C", call["args"]) + self.assertEqual(call["args"][call["args"].index("-F") + 1], "db.example.org") + + def test_invalid_tls_configuration_fails_before_connecting(self): + result = self.run_job(MSSQL_TRUST_SERVER_CERTIFICATE="typo") + self.assertNotEqual(result.returncode, 0) + self.assertEqual(self.calls(), []) + + def test_ingestion_password_is_escaped_on_stdin_without_sqlcmd_expansion(self): + password = "fixture'$(literal)password" + result = self.run_job(COGSTACK_INGEST_PASSWORD=password) + self.assertEqual(result.returncode, 0, result.stderr) + call = self.calls()[-1] + self.assertIn("-x", call["args"]) + self.assertNotIn(password, call["args"]) + self.assertIn("N'fixture''$(literal)password'", call["stdin"]) + + def test_access_provisioning_failure_fails_the_job(self): + result = self.run_job(FAKE_ACCESS_FAILURE="1") + self.assertEqual(result.returncode, 7) + self.assertNotIn("initialization completed", result.stdout) + if __name__ == "__main__": unittest.main() diff --git a/scripts/tests/test_opensearch_deployment.py b/scripts/tests/test_opensearch_deployment.py index 5487572b0..b03a5ba6c 100755 --- a/scripts/tests/test_opensearch_deployment.py +++ b/scripts/tests/test_opensearch_deployment.py @@ -246,6 +246,14 @@ def test_node_certificates_verify_for_compose_and_helm_service_names(self) -> No key = cert.with_suffix(".key") self.command("openssl", "pkey", "-in", str(key), "-check", "-noout") + def test_shared_ca_signing_keys_are_not_copied_into_the_opensearch_tree(self) -> None: + self.assertTrue((self.certificates / "elastic-stack-ca.crt.pem").is_file()) + self.assertFalse((self.certificates / "elastic-stack-ca.key.pem").exists()) + self.assertFalse((self.certificates / "elastic-stack-ca.p12").exists()) + root_ca = self.workspace / "security/certificates/root/root-ca" + self.assertTrue(root_ca.with_suffix(".key").is_file()) + self.assertTrue(root_ca.with_suffix(".p12").is_file()) + def test_dashboards_certificate_verifies_for_localhost_and_kubernetes(self) -> None: cert = self.certificates / "es_kibana_client.pem" ca = self.certificates / "elastic-stack-ca.crt.pem" diff --git a/scripts/tests/test_postgres_certificates.py b/scripts/tests/test_postgres_certificates.py new file mode 100644 index 000000000..98fbd858c --- /dev/null +++ b/scripts/tests/test_postgres_certificates.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python3 +"""Exercise PostgreSQL certificate provisioning with isolated OpenSSL fixtures.""" + +from __future__ import annotations + +import os +import shlex +import shutil +import stat +import subprocess +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +HELPER = ROOT / "security/scripts/create_postgres_certs.sh" + + +@unittest.skipUnless(shutil.which("openssl"), "OpenSSL is required") +class PostgresCertificateTest(unittest.TestCase): + def setUp(self) -> None: + self.temp = tempfile.TemporaryDirectory(prefix="cogstack postgres tls ") + self.addCleanup(self.temp.cleanup) + self.directory = Path(self.temp.name) + self.profile = self.directory / "profile with spaces" + (self.profile / "security").mkdir(parents=True) + self.certificates = self.directory / "certificates" + self.samples = self.directory / "samples tls" + self.databank = self.directory / "databank tls" + self.other_cwd = self.directory / "unrelated working directory" + self.other_cwd.mkdir() + self.env = { + key: value for key, value in os.environ.items() + if not key.startswith(("COGSTACK_", "POSTGRES_", "ROOT_CERTIFICATE_", "SECURITY_")) + } + self.env["COGSTACK_CONFIG_DIR"] = str(self.profile) + (self.profile / "security/certificates_general.env").write_text( + f"SECURITY_CERTIFICATES_DIR={shlex.quote(str(self.certificates))}\n" + "ROOT_CERTIFICATE_NAME=fixture-ca\n" + "ROOT_CERTIFICATE_TIME_VAILIDITY_IN_DAYS=30\n" + ) + self.configure_outputs(self.samples, self.databank) + + def configure_outputs(self, samples: Path | str, databank: Path | str) -> None: + (self.profile / "database.env").write_text( + f"POSTGRES_SAMPLES_TLS_DIR={shlex.quote(str(samples))}\n" + f"POSTGRES_DATABANK_TLS_DIR={shlex.quote(str(databank))}\n" + ) + + def openssl(self, *args: str) -> str: + result = subprocess.run( + ["openssl", *args], cwd=self.other_cwd, env=self.env, + text=True, capture_output=True, timeout=30, + ) + self.assertEqual(result.returncode, 0, result.stderr) + return result.stdout + + def create_signing_ca(self) -> Path: + root = self.certificates / "root" + root.mkdir(parents=True) + certificate = root / "fixture-ca.pem" + self.openssl( + "req", "-new", "-x509", "-newkey", "rsa:2048", "-nodes", "-sha256", + "-keyout", str(root / "fixture-ca.key"), "-out", str(certificate), + "-days", "2", "-subj", "/CN=PostgreSQL fixture signing CA", + "-addext", "basicConstraints=critical,CA:TRUE", + "-addext", "keyUsage=critical,keyCertSign,cRLSign", + ) + return certificate + + def provision(self) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["bash", str(HELPER)], cwd=self.other_cwd, env=self.env, + text=True, capture_output=True, timeout=90, + ) + + @staticmethod + def snapshot(directory: Path) -> dict[str, tuple[bytes | None, int, int]]: + return { + str(path.relative_to(directory)): ( + path.read_bytes() if path.is_file() else None, + stat.S_IMODE(path.stat().st_mode), path.stat().st_mtime_ns, + ) + for path in (directory, *directory.rglob("*")) + } + + @staticmethod + def supplied_bundle(directory: Path) -> None: + directory.mkdir() + for name in ("server.crt", "server.key", "ca.crt"): + (directory / name).write_text(f"Externally supplied {name}\n") + (directory / "operator-note.txt").write_text("Keep this bundle unchanged.\n") + + def test_generated_certificates_verify_for_service_names_and_have_safe_permissions(self) -> None: + ca = self.create_signing_ca() + result = self.provision() + self.assertEqual(result.returncode, 0, result.stderr) + for bundle, hostnames in ( + (self.samples, ("samples-db", "cogstack-samples-db", "localhost")), + (self.databank, ("cogstack-databank-db", "cogstack-production-databank-db", "localhost")), + ): + with self.subTest(bundle=bundle.name): + cert, key = bundle / "server.crt", bundle / "server.key" + self.assertEqual((bundle / "ca.crt").read_bytes(), ca.read_bytes()) + for hostname in hostnames: + self.openssl( + "verify", "-CAfile", str(bundle / "ca.crt"), "-purpose", "sslserver", + "-verify_hostname", hostname, str(cert), + ) + self.openssl( + "verify", "-CAfile", str(bundle / "ca.crt"), + "-verify_ip", "127.0.0.1", str(cert), + ) + public_key = self.openssl("pkey", "-in", str(key), "-pubout") + self.assertEqual(public_key, self.openssl("x509", "-in", str(cert), "-pubkey", "-noout")) + self.assertEqual(stat.S_IMODE(bundle.stat().st_mode), 0o755) + self.assertEqual(stat.S_IMODE(key.stat().st_mode), 0o600) + for name in ("server.crt", "ca.crt"): + self.assertEqual(stat.S_IMODE((bundle / name).stat().st_mode), 0o644) + + def test_complete_supplied_bundles_are_unchanged_without_signing_ca(self) -> None: + for bundle in (self.samples, self.databank): + self.supplied_bundle(bundle) + before = [self.snapshot(bundle) for bundle in (self.samples, self.databank)] + result = self.provision() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(before, [self.snapshot(bundle) for bundle in (self.samples, self.databank)]) + self.assertFalse(self.certificates.exists()) + + def test_partial_or_nonempty_directories_are_rejected_without_changes(self) -> None: + self.create_signing_ca() + for invalid_name in ("server.crt", "operator-note.txt"): + for invalid_service in ("samples", "databank"): + with self.subTest(file=invalid_name, service=invalid_service): + samples = self.directory / f"samples-{invalid_service}-{invalid_name}" + databank = self.directory / f"databank-{invalid_service}-{invalid_name}" + invalid = samples if invalid_service == "samples" else databank + supplied = databank if invalid_service == "samples" else samples + invalid.mkdir() + (invalid / invalid_name).write_text("Deployment-managed input\n") + self.supplied_bundle(supplied) + self.configure_outputs(samples, databank) + before = [self.snapshot(bundle) for bundle in (samples, databank)] + result = self.provision() + self.assertNotEqual(result.returncode, 0, result.stdout) + self.assertEqual(before, [self.snapshot(bundle) for bundle in (samples, databank)]) + + def test_relative_output_paths_resolve_from_deploy_instead_of_working_directory(self) -> None: + self.create_signing_ca() + self.configure_outputs( + os.path.relpath(self.samples, ROOT / "deploy"), + os.path.relpath(self.databank, ROOT / "deploy"), + ) + result = self.provision() + self.assertEqual(result.returncode, 0, result.stderr) + for bundle in (self.samples, self.databank): + self.assertTrue((bundle / "server.key").is_file()) + self.openssl("verify", "-CAfile", str(bundle / "ca.crt"), str(bundle / "server.crt")) + + @unittest.skipUnless(shutil.which("make"), "Make is required") + def test_make_target_uses_existing_ca_with_a_certificate_path_containing_spaces(self) -> None: + ca = self.create_signing_ca() + for suffix in (".p12", "-keystore.jks", "-truststore.jks"): + (ca.parent / f"fixture-ca{suffix}").touch() + before = self.snapshot(ca.parent) + # Exercise the Make argument itself; a profile assignment would mask + # incorrect path handling by replacing the value inside the helper. + general = self.profile / "security/certificates_general.env" + general.write_text("\n".join( + line for line in general.read_text().splitlines() + if not line.startswith("SECURITY_CERTIFICATES_DIR=") + ) + "\n") + result = subprocess.run( + ["make", "-C", str(ROOT / "deploy"), "init-security-postgresql", + f"SECURITY_CERTIFICATES_DIR={self.certificates}"], + cwd=self.other_cwd, env=self.env, text=True, capture_output=True, timeout=90, + ) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(before, self.snapshot(ca.parent)) + for bundle, hostname in ( + (self.samples, "samples-db"), (self.databank, "cogstack-databank-db"), + ): + with self.subTest(bundle=bundle.name): + self.assertTrue((bundle / "server.key").is_file()) + self.assertEqual(ca.read_bytes(), (bundle / "ca.crt").read_bytes()) + self.openssl( + "verify", "-CAfile", str(ca), "-purpose", "sslserver", + "-verify_hostname", hostname, str(bundle / "server.crt"), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tests/test_postgres_initialization.py b/scripts/tests/test_postgres_initialization.py index ba17944d0..733aa61e2 100755 --- a/scripts/tests/test_postgres_initialization.py +++ b/scripts/tests/test_postgres_initialization.py @@ -39,7 +39,7 @@ def setUp(self): from pathlib import Path args = sys.argv[1:] -stdin = "" if "-f" in args else sys.stdin.read() +stdin = "" if "-f" in args or "-c" in args else sys.stdin.read() with Path(os.environ["FAKE_PSQL_LOG"]).open("a") as stream: stream.write(json.dumps({"args": args, "stdin": stdin}) + "\\n") if "-f" in args and os.environ.get("FAKE_FAIL_FILE") == Path(args[args.index("-f") + 1]).name: @@ -53,11 +53,15 @@ def setUp(self): "POSTGRES_USER": "site user", "POSTGRES_DB": "site database", "POSTGRES_DB_SCHEMA_PREFIX": "cogstack_db", + "POSTGRES_PASSWORD": "fixture-bootstrap-password", + "COGSTACK_INGEST_USER": "fixture_ingest", + "COGSTACK_INGEST_PASSWORD": "fixture-ingest-password", "FAKE_PSQL_LOG": str(self.log), } def install_samples_bundle(self): shutil.copy2(ROOT / "services/pgsamples/init_db.sh", self.script) + shutil.copytree(ROOT / "services/pgsamples/access", self.bundle / "access") (self.schemas / "samples_db_schema.sql").write_text("-- sample schema\n") for name in ("encounters_pdf_text_small.sql", "encounters_docx_small.sql"): (self.dumps / name).write_text("-- " + name + "\n") @@ -70,6 +74,7 @@ def install_samples_bundle(self): def install_databank_bundle(self): shutil.copy2(ROOT / "services/cogstack-db/pgsql/init_db.sh", self.script) + shutil.copytree(ROOT / "services/cogstack-db/pgsql/access", self.bundle / "access") for name in ( "annotations_nlp_create_schema.sql", "cogstack_db_02 second.sql", @@ -95,7 +100,7 @@ def test_samples_load_all_inputs_from_the_bundle_into_postgres_db(self): result = self.run_loader() self.assertEqual(result.returncode, 0, result.stderr) calls = self.calls() - self.assertEqual(len(calls), 6) + self.assertEqual(len(calls), 8) self.assertEqual(Path(calls[0]["args"][-1]), self.schemas / "samples_db_schema.sql") self.assertIn("samples_db_data.sql.gz", calls[1]["stdin"]) self.assertEqual(Path(calls[2]["args"][-1]).name, "encounters_pdf_text_small.sql") @@ -136,10 +141,10 @@ def test_databank_selects_custom_sql_in_order_without_creating_database(self): self.assertEqual(result.returncode, 0, result.stderr) calls = self.calls() self.assertEqual( - [Path(c["args"][-1]).name for c in calls], + [Path(c["args"][-1]).name for c in calls[:3]], ["annotations_nlp_create_schema.sql", "cogstack_db_01 first.sql", "cogstack_db_02 second.sql"], ) - for call in calls: + for call in calls[:3]: self.assertEqual(Path(call["args"][-1]).parent, self.schemas) self.assertEqual(call["args"][call["args"].index("-d") + 1], "site database") self.assertNotIn("CREATE DATABASE", call["stdin"]) diff --git a/security/env/users_database.env b/security/env/users_database.env index b2140eccf..c8e8c16ee 100644 --- a/security/env/users_database.env +++ b/security/env/users_database.env @@ -8,4 +8,13 @@ DATABASE_PASSWORD=admin # Production DATABASE MSSQL user MSSQL_SA_USER=sa -MSSQL_SA_PASSWORD=admin!COGSTACK2022 \ No newline at end of file +MSSQL_SA_PASSWORD=admin!COGSTACK2022 + +# Restricted application accounts. Public development passwords only. +# Bootstrap accounts above are reserved for database initialization/administration. +POSTGRES_INGEST_USER_SAMPLES=cogstack_samples +POSTGRES_INGEST_PASSWORD_SAMPLES=SamplesIngest!1 +DATABASE_INGEST_USER=cogstack_ingest +DATABASE_INGEST_PASSWORD=DatabankIngest!1 +MSSQL_INGEST_USER=cogstack_ingest +MSSQL_INGEST_PASSWORD=MssqlIngest!1 diff --git a/security/scripts/create_opensearch_node_cert.sh b/security/scripts/create_opensearch_node_cert.sh index 94679598b..3904bfbd1 100755 --- a/security/scripts/create_opensearch_node_cert.sh +++ b/security/scripts/create_opensearch_node_cert.sh @@ -70,7 +70,6 @@ OPENSEARCH_FOLDER="${SECURITY_CERTIFICATES_FOLDER}elastic/opensearch/" ES_CERTIFICATES_FOLDER="${OPENSEARCH_FOLDER}elasticsearch/" CA_ROOT_CERT="${ROOT_CERTIFICATES_FOLDER}${ROOT_CERTIFICATE_NAME}.pem" CA_ROOT_KEY="${ROOT_CERTIFICATES_FOLDER}${ROOT_CERTIFICATE_NAME}.key" -CA_ROOT_KEYSTORE="${ROOT_CERTIFICATES_FOLDER}${ROOT_CERTIFICATE_NAME}.p12" # === Validate Root CA Files === if [[ ! -f "$CA_ROOT_CERT" || ! -f "$CA_ROOT_KEY" ]]; then @@ -145,11 +144,9 @@ for cert_name in "${CERT_NAMES[@]}"; do generate_node_cert "$cert_name" done -# === Copy Root CA files to opensearch folder === +# === Copy only the public root CA certificate to the served tree === ELASTIC_CA_PREFIX="elastic-stack-ca" -cp "$CA_ROOT_KEY" "${OPENSEARCH_FOLDER}${ELASTIC_CA_PREFIX}.key.pem" cp "$CA_ROOT_CERT" "${OPENSEARCH_FOLDER}${ELASTIC_CA_PREFIX}.crt.pem" -cp "$CA_ROOT_KEYSTORE" "${OPENSEARCH_FOLDER}${ELASTIC_CA_PREFIX}.p12" # Ensure generated cert artifacts are readable and directories are traversable. # Note: certificate/key files should not be executable. diff --git a/security/scripts/create_postgres_certs.sh b/security/scripts/create_postgres_certs.sh new file mode 100755 index 000000000..d89064c21 --- /dev/null +++ b/security/scripts/create_postgres_certs.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +# Generate PostgreSQL leaf certificates without replacing supplied bundles. +set -euo pipefail +umask 077 + +# shellcheck disable=SC1090,SC1091 +source "$(dirname "${BASH_SOURCE[0]}")/load_config.sh" +source "$(cogstack_env_file deploy/database.env)" +source "$(cogstack_env_file security/env/certificates_general.env)" + +resolve_directory() { + case "$1" in + /*) printf '%s\n' "$1" ;; + *) printf '%s/deploy/%s\n' "$COGSTACK_REPO_ROOT" "$1" ;; + esac +} + +samples_dir="$(resolve_directory "${POSTGRES_SAMPLES_TLS_DIR:-../security/certificates/postgresql/samples}")" +databank_dir="$(resolve_directory "${POSTGRES_DATABANK_TLS_DIR:-../security/certificates/postgresql/databank}")" +missing_dirs=() +for directory in "$samples_dir" "$databank_dir"; do + if [[ -s "$directory/server.crt" && -s "$directory/server.key" && -s "$directory/ca.crt" ]]; then + echo "PostgreSQL TLS bundle already exists: $directory; leaving it unchanged." + elif [[ -e "$directory" && ( ! -d "$directory" || -n "$(find "$directory" -mindepth 1 -maxdepth 1 -print -quit)" ) ]]; then + echo "ERROR: $directory contains an incomplete PostgreSQL TLS bundle; preserve and repair it before retrying." >&2 + exit 1 + else + missing_dirs+=("$directory") + fi +done +if [[ ${#missing_dirs[@]} == 0 ]]; then exit 0; fi +if [[ "$samples_dir" == "$databank_dir" ]]; then + echo "ERROR: generated PostgreSQL TLS bundles require separate output directories." >&2 + exit 1 +fi + +root_dir="$(resolve_directory "${SECURITY_CERTIFICATES_DIR:-../security/certificates}")/root" +ca_cert="$root_dir/${ROOT_CERTIFICATE_NAME:?Set ROOT_CERTIFICATE_NAME}.pem" +ca_key="$root_dir/$ROOT_CERTIFICATE_NAME.key" +if [[ ! -s "$ca_cert" || ! -s "$ca_key" ]]; then + echo "ERROR: shared root CA is missing; run 'make -C deploy init-security-root-ca' first." >&2 + exit 1 +fi + +work_dir="$(mktemp -d)" +trap 'rm -rf "$work_dir"' EXIT +generate_bundle() { + local directory="$1" hostname="$2" alias="$3" + if [[ -s "$directory/server.crt" && -s "$directory/server.key" && -s "$directory/ca.crt" ]]; then return; fi + openssl req -new -newkey rsa:2048 -nodes -sha256 -subj "/CN=$hostname" \ + -keyout "$work_dir/server.key" -out "$work_dir/server.csr" + cat > "$work_dir/extensions.cnf" < 256 OR DATALENGTH(@ingest_password) > 256 +BEGIN + THROW 51000, 'The ingestion login or password exceeds SQL Server length limits.', 1; +END; +BEGIN TRANSACTION; + +-- Never turn an administrator, database owner, or existing role member into +-- the application account. Choose a separate SQL login for ingestion. +IF @ingest_user = ORIGINAL_LOGIN() + OR EXISTS ( + SELECT 1 FROM sys.server_role_members + WHERE member_principal_id = SUSER_ID(@ingest_user) + ) + OR EXISTS (SELECT 1 FROM sys.databases WHERE owner_sid = SUSER_SID(@ingest_user)) + OR EXISTS ( + SELECT 1 FROM sys.server_permissions + WHERE grantee_principal_id = SUSER_ID(@ingest_user) + AND state IN ('G', 'W') AND permission_name <> 'CONNECT SQL' + ) + OR EXISTS ( + SELECT 1 FROM sys.server_principals + WHERE name = @ingest_user AND type <> 'S' + ) +BEGIN + THROW 51000, 'Choose a separate, unprivileged SQL ingestion login.', 1; +END; + +DECLARE @login_sql nvarchar(max); +IF SUSER_ID(@ingest_user) IS NULL + SET @login_sql = N'CREATE LOGIN ' + QUOTENAME(@ingest_user) + + N' WITH PASSWORD = N' + QUOTENAME(@ingest_password, '''') + N', CHECK_POLICY = ON'; +ELSE + SET @login_sql = N'ALTER LOGIN ' + QUOTENAME(@ingest_user) + + N' WITH PASSWORD = N' + QUOTENAME(@ingest_password, ''''); +EXEC sys.sp_executesql @login_sql; + +IF USER_ID(@ingest_user) IS NOT NULL +BEGIN + IF EXISTS ( + SELECT 1 FROM sys.database_role_members + WHERE member_principal_id = USER_ID(@ingest_user) + ) OR EXISTS ( + SELECT 1 FROM sys.schemas WHERE principal_id = USER_ID(@ingest_user) + ) OR EXISTS ( + SELECT 1 FROM sys.objects WHERE principal_id = USER_ID(@ingest_user) + ) OR EXISTS ( + SELECT 1 FROM sys.database_principals + WHERE name = @ingest_user AND sid <> SUSER_SID(@ingest_user) + ) OR EXISTS ( + SELECT 1 FROM sys.database_permissions + WHERE grantee_principal_id = USER_ID(@ingest_user) + AND state IN ('G', 'W') + AND permission_name NOT IN ('CONNECT', 'SELECT', 'INSERT', 'UPDATE', 'DELETE') + ) + BEGIN + THROW 51000, 'The ingestion database user has ownership, role membership, or a different login mapping.', 1; + END; +END +ELSE +BEGIN + DECLARE @user_sql nvarchar(max) = N'CREATE USER ' + QUOTENAME(@ingest_user) + + N' FOR LOGIN ' + QUOTENAME(@ingest_user); + EXEC sys.sp_executesql @user_sql; +END; + +DECLARE @grant_sql nvarchar(max) = N'GRANT SELECT, INSERT, UPDATE, DELETE ON SCHEMA::[CogStack] TO ' + + QUOTENAME(@ingest_user); +EXEC sys.sp_executesql @grant_sql; +COMMIT TRANSACTION; diff --git a/services/cogstack-db/mssql/init_db.sh b/services/cogstack-db/mssql/init_db.sh index bf25a8143..a457a6c21 100755 --- a/services/cogstack-db/mssql/init_db.sh +++ b/services/cogstack-db/mssql/init_db.sh @@ -12,9 +12,25 @@ annotations_schema="$data_dir/annotations_nlp_create_schema.sql" retries="${MSSQL_INIT_RETRIES:-60}" retry_delay="${MSSQL_INIT_RETRY_DELAY_SECONDS:-2}" sqlcmd_bin="${SQLCMD_BIN:-}" +trust_server_certificate="${MSSQL_TRUST_SERVER_CERTIFICATE:-true}" +access_sql="$bootstrap_dir/access/ingest.sql" : "${MSSQL_SA_PASSWORD:?Set MSSQL_SA_PASSWORD before initializing SQL Server}" export SQLCMDPASSWORD="$MSSQL_SA_PASSWORD" +: "${COGSTACK_INGEST_USER:?Set a separate MSSQL ingestion user}" +: "${COGSTACK_INGEST_PASSWORD:?Set the MSSQL ingestion password}" +if [[ ${#COGSTACK_INGEST_USER} -gt 128 || ${#COGSTACK_INGEST_PASSWORD} -gt 128 ]]; then + echo "MSSQL ingestion usernames and passwords must not exceed 128 characters." >&2 + exit 1 +fi +if [[ "$trust_server_certificate" != true && "$trust_server_certificate" != false ]]; then + echo "MSSQL_TRUST_SERVER_CERTIFICATE must be true or false." >&2 + exit 1 +fi +if [[ ! -r "$access_sql" ]]; then + echo "Missing ingestion access SQL: $access_sql" >&2 + exit 1 +fi if [[ ! "$retries" =~ ^[1-9][0-9]*$ || ! "$retry_delay" =~ ^[0-9]+$ ]]; then echo "MSSQL_INIT_RETRIES must be positive and MSSQL_INIT_RETRY_DELAY_SECONDS non-negative integers." >&2 @@ -42,9 +58,15 @@ if [[ ! -x "$sqlcmd_bin" ]]; then exit 1 fi -# The reference Compose server uses its self-signed certificate. -b propagates -# SQL errors to the job exit status; the password stays out of command arguments. -sqlcmd=("$sqlcmd_bin" -S "$server" -U "$db_user" -C -b -l 5) +# Always require encryption. Trust bypass is explicit for the local self-signed +# server; verified mode never falls back to -C. Keep passwords out of arguments. +sqlcmd=("$sqlcmd_bin" -S "$server" -U "$db_user" -N -b -l 5) +if [[ "$trust_server_certificate" == true ]]; then + sqlcmd+=(-C) +fi +if [[ -n "${MSSQL_TLS_HOSTNAME:-}" ]]; then + sqlcmd+=(-F "$MSSQL_TLS_HOSTNAME") +fi for ((attempt = 1; attempt <= retries; attempt++)); do if "${sqlcmd[@]}" -d master -Q 'SET NOCOUNT ON; SELECT 1;' >/dev/null 2>&1; then break @@ -78,4 +100,14 @@ done echo "Applying annotation schema: $annotations_schema" "${sqlcmd[@]}" -d "$db_name" -i "$annotations_schema" + +# Send escaped values on stdin; -x prevents sqlcmd variable expansion inside +# passwords (for example a literal $(...)). The SQL file contains no secrets. +ingest_user_sql="${COGSTACK_INGEST_USER//$single_quote/$single_quote$single_quote}" +ingest_password_sql="${COGSTACK_INGEST_PASSWORD//$single_quote/$single_quote$single_quote}" +{ + printf "DECLARE @ingest_user nvarchar(max) = N'%s';\n" "$ingest_user_sql" + printf "DECLARE @ingest_password nvarchar(max) = N'%s';\n" "$ingest_password_sql" + cat "$access_sql" +} | "${sqlcmd[@]}" -d "$db_name" -x echo "SQL Server database initialization completed." diff --git a/services/cogstack-db/pgsql/access/configure.sh b/services/cogstack-db/pgsql/access/configure.sh new file mode 100755 index 000000000..a6327eed0 --- /dev/null +++ b/services/cogstack-db/pgsql/access/configure.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# Also usable against an existing volume via docker compose exec. +set -euo pipefail + +access_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +: "${POSTGRES_USER:?Set the PostgreSQL bootstrap user}" +: "${POSTGRES_DB:?Set the target database}" +: "${COGSTACK_INGEST_USER:?Set a separate ingestion user}" +: "${COGSTACK_INGEST_PASSWORD:?Set the ingestion password}" +export PGPASSWORD="${POSTGRES_PASSWORD:?Set the PostgreSQL bootstrap password}" + +psql_command=(psql -X -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB") +for ((attempt = 1; attempt <= 60; attempt++)); do + if "${psql_command[@]}" -c 'SELECT 1' >/dev/null 2>&1; then + exec "${psql_command[@]}" -f "$access_dir/ingest.sql" + fi + if ((attempt == 60)); then + echo "PostgreSQL database $POSTGRES_DB did not become ready." >&2 + exit 1 + fi + sleep 2 +done diff --git a/services/cogstack-db/pgsql/access/ingest.sql b/services/cogstack-db/pgsql/access/ingest.sql new file mode 100644 index 000000000..194a1051c --- /dev/null +++ b/services/cogstack-db/pgsql/access/ingest.sql @@ -0,0 +1,66 @@ +\set ON_ERROR_STOP on +\getenv ingest_user COGSTACK_INGEST_USER +\getenv ingest_password COGSTACK_INGEST_PASSWORD +BEGIN; +SELECT set_config('cogstack.ingest_user', :'ingest_user', true); + +-- A configured application login must never reuse an administrator, owner, +-- or a role with inherited permissions. +DO $guard$ +DECLARE + app_name text := current_setting('cogstack.ingest_user'); + app_oid oid := (SELECT oid FROM pg_roles WHERE rolname = app_name); +BEGIN + IF octet_length(app_name) > 63 THEN + RAISE EXCEPTION 'The ingestion login exceeds PostgreSQL identifier length limits'; + END IF; + IF app_name = current_user OR EXISTS ( + SELECT 1 FROM pg_roles + WHERE oid = app_oid AND (rolsuper OR rolcreatedb OR rolcreaterole OR rolreplication OR rolbypassrls) + ) OR EXISTS ( + SELECT 1 FROM pg_auth_members WHERE member = app_oid + ) OR EXISTS ( + SELECT 1 FROM pg_database WHERE datdba = app_oid + ) OR EXISTS ( + SELECT 1 FROM pg_namespace WHERE nspowner = app_oid + ) OR EXISTS ( + SELECT 1 FROM pg_class WHERE relowner = app_oid + ) THEN + RAISE EXCEPTION 'Choose a separate, unprivileged ingestion login instead of %', app_name; + END IF; +END +$guard$; + +SELECT format( + 'CREATE ROLE %I LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT NOREPLICATION NOBYPASSRLS', + :'ingest_user' +) WHERE NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = :'ingest_user') +\gexec +SELECT format('ALTER ROLE %I LOGIN PASSWORD %L', :'ingest_user', :'ingest_password') +\gexec +SELECT format('GRANT CONNECT ON DATABASE %I TO %I', current_database(), :'ingest_user') +\gexec +GRANT USAGE ON SCHEMA public TO :"ingest_user"; +SELECT format('REVOKE CREATE ON DATABASE %I FROM %I', current_database(), :'ingest_user') +\gexec +REVOKE CREATE ON SCHEMA public FROM :"ingest_user"; +REVOKE ALL PRIVILEGES ON ALL TABLES IN SCHEMA public FROM :"ingest_user"; +REVOKE ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public FROM :"ingest_user"; +ALTER DEFAULT PRIVILEGES IN SCHEMA public REVOKE ALL ON TABLES FROM :"ingest_user"; +ALTER DEFAULT PRIVILEGES IN SCHEMA public REVOKE ALL ON SEQUENCES FROM :"ingest_user"; +GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO :"ingest_user"; +GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA public TO :"ingest_user"; +ALTER DEFAULT PRIVILEGES IN SCHEMA public + GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO :"ingest_user"; +ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT USAGE, SELECT ON SEQUENCES TO :"ingest_user"; + +-- Fail rather than silently inheriting CREATE from legacy PUBLIC grants. +DO $guard$ +BEGIN + IF has_schema_privilege(current_setting('cogstack.ingest_user'), 'public', 'CREATE') + OR has_database_privilege(current_setting('cogstack.ingest_user'), current_database(), 'CREATE') THEN + RAISE EXCEPTION 'The ingestion login inherits CREATE privileges; remove legacy PUBLIC CREATE grants first'; + END IF; +END +$guard$; +COMMIT; diff --git a/services/cogstack-db/pgsql/access/pg_hba_tls.conf b/services/cogstack-db/pgsql/access/pg_hba_tls.conf new file mode 100644 index 000000000..12293a447 --- /dev/null +++ b/services/cogstack-db/pgsql/access/pg_hba_tls.conf @@ -0,0 +1,3 @@ +local all all trust +hostnossl all all all reject +hostssl all all all scram-sha-256 diff --git a/services/cogstack-db/pgsql/init_db.sh b/services/cogstack-db/pgsql/init_db.sh index 9c3a2ffb2..7c0abf62f 100755 --- a/services/cogstack-db/pgsql/init_db.sh +++ b/services/cogstack-db/pgsql/init_db.sh @@ -26,4 +26,5 @@ for schema_file in "${schema_files[@]}"; do "${psql_command[@]}" -f "$schema_file" done +"$bootstrap_dir/access/configure.sh" echo "PostgreSQL databank initialization completed." diff --git a/services/pgsamples/access/configure.sh b/services/pgsamples/access/configure.sh new file mode 100755 index 000000000..a6327eed0 --- /dev/null +++ b/services/pgsamples/access/configure.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# Also usable against an existing volume via docker compose exec. +set -euo pipefail + +access_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +: "${POSTGRES_USER:?Set the PostgreSQL bootstrap user}" +: "${POSTGRES_DB:?Set the target database}" +: "${COGSTACK_INGEST_USER:?Set a separate ingestion user}" +: "${COGSTACK_INGEST_PASSWORD:?Set the ingestion password}" +export PGPASSWORD="${POSTGRES_PASSWORD:?Set the PostgreSQL bootstrap password}" + +psql_command=(psql -X -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB") +for ((attempt = 1; attempt <= 60; attempt++)); do + if "${psql_command[@]}" -c 'SELECT 1' >/dev/null 2>&1; then + exec "${psql_command[@]}" -f "$access_dir/ingest.sql" + fi + if ((attempt == 60)); then + echo "PostgreSQL database $POSTGRES_DB did not become ready." >&2 + exit 1 + fi + sleep 2 +done diff --git a/services/pgsamples/access/ingest.sql b/services/pgsamples/access/ingest.sql new file mode 100644 index 000000000..8e9d15e57 --- /dev/null +++ b/services/pgsamples/access/ingest.sql @@ -0,0 +1,63 @@ +\set ON_ERROR_STOP on +\getenv ingest_user COGSTACK_INGEST_USER +\getenv ingest_password COGSTACK_INGEST_PASSWORD +BEGIN; +SELECT set_config('cogstack.ingest_user', :'ingest_user', true); + +-- A configured application login must never reuse an administrator, owner, +-- or a role with inherited permissions. +DO $guard$ +DECLARE + app_name text := current_setting('cogstack.ingest_user'); + app_oid oid := (SELECT oid FROM pg_roles WHERE rolname = app_name); +BEGIN + IF octet_length(app_name) > 63 THEN + RAISE EXCEPTION 'The ingestion login exceeds PostgreSQL identifier length limits'; + END IF; + IF app_name = current_user OR EXISTS ( + SELECT 1 FROM pg_roles + WHERE oid = app_oid AND (rolsuper OR rolcreatedb OR rolcreaterole OR rolreplication OR rolbypassrls) + ) OR EXISTS ( + SELECT 1 FROM pg_auth_members WHERE member = app_oid + ) OR EXISTS ( + SELECT 1 FROM pg_database WHERE datdba = app_oid + ) OR EXISTS ( + SELECT 1 FROM pg_namespace WHERE nspowner = app_oid + ) OR EXISTS ( + SELECT 1 FROM pg_class WHERE relowner = app_oid + ) THEN + RAISE EXCEPTION 'Choose a separate, unprivileged ingestion login instead of %', app_name; + END IF; +END +$guard$; + +SELECT format( + 'CREATE ROLE %I LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT NOREPLICATION NOBYPASSRLS', + :'ingest_user' +) WHERE NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = :'ingest_user') +\gexec +SELECT format('ALTER ROLE %I LOGIN PASSWORD %L', :'ingest_user', :'ingest_password') +\gexec +SELECT format('GRANT CONNECT ON DATABASE %I TO %I', current_database(), :'ingest_user') +\gexec +GRANT USAGE ON SCHEMA public TO :"ingest_user"; +SELECT format('REVOKE CREATE ON DATABASE %I FROM %I', current_database(), :'ingest_user') +\gexec +REVOKE CREATE ON SCHEMA public FROM :"ingest_user"; +REVOKE ALL PRIVILEGES ON ALL TABLES IN SCHEMA public FROM :"ingest_user"; +REVOKE ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public FROM :"ingest_user"; +ALTER DEFAULT PRIVILEGES IN SCHEMA public REVOKE ALL ON TABLES FROM :"ingest_user"; +ALTER DEFAULT PRIVILEGES IN SCHEMA public REVOKE ALL ON SEQUENCES FROM :"ingest_user"; +GRANT SELECT ON ALL TABLES IN SCHEMA public TO :"ingest_user"; +ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO :"ingest_user"; + +-- Fail rather than silently inheriting CREATE from legacy PUBLIC grants. +DO $guard$ +BEGIN + IF has_schema_privilege(current_setting('cogstack.ingest_user'), 'public', 'CREATE') + OR has_database_privilege(current_setting('cogstack.ingest_user'), current_database(), 'CREATE') THEN + RAISE EXCEPTION 'The ingestion login inherits CREATE privileges; remove legacy PUBLIC CREATE grants first'; + END IF; +END +$guard$; +COMMIT; diff --git a/services/pgsamples/access/pg_hba_tls.conf b/services/pgsamples/access/pg_hba_tls.conf new file mode 100644 index 000000000..afd58160a --- /dev/null +++ b/services/pgsamples/access/pg_hba_tls.conf @@ -0,0 +1,5 @@ +# Preserve local bootstrap/administration over the container's Unix socket. +local all all trust +# Network clients must use TLS and password authentication. +hostnossl all all all reject +hostssl all all all scram-sha-256 diff --git a/services/pgsamples/init_db.sh b/services/pgsamples/init_db.sh index 3cd6bac6f..f179e8695 100755 --- a/services/pgsamples/init_db.sh +++ b/services/pgsamples/init_db.sh @@ -41,4 +41,5 @@ for gzip_file in "${extra_gzip_files[@]}"; do gunzip -c "$gzip_file" | "${psql_command[@]}" done +"$bootstrap_dir/access/configure.sh" echo "Sample database initialization completed." From 7937f0bfe4fde0b05f14301bba45cba44c278807 Mon Sep 17 00:00:00 2001 From: vladd-bit Date: Tue, 10 Nov 2026 13:03:27 +0000 Subject: [PATCH 24/28] Refactor NiFi/nginx runtimes and simplify Compose configuration - Reuse Apache NiFi startup defaults and remove redundant NAR downloads. - Preserve editable processor mounts and separate generated state from source. - Support non-root execution, health probes and graceful shutdown. - Improve nginx TLS verification, DNS resolution and runtime configuration. - Group Compose environment, mount and network anchors. - Consolidate NIFI_ settings in nifi.env. - Update credential setup, existing smoke tests, CI and deployment docs. --- .dockerignore | 4 + .github/workflows/compose_config.yml | 7 + .github/workflows/smoke_nifi_services.yml | 10 +- deploy/Makefile | 4 +- deploy/export_env_vars.sh | 6 + deploy/nginx.env | 13 +- deploy/nifi.env | 21 +- deploy/services.dev.yml | 270 +++--- deploy/services.yml | 813 ++++++++++-------- docs/deploy/deployment.md | 4 + docs/deploy/nifi-runtime.md | 174 ++++ docs/security/nifi.md | 78 +- mkdocs.yml | 1 + nifi/Dockerfile | 153 ++-- nifi/conf/nifi.properties | 14 +- nifi/docker/healthcheck.sh | 17 + scripts/tests/smoke_nifi_services.sh | 12 +- scripts/tests/test_nifi_runtime.py | 73 ++ .../scripts/nifi_init_create_user_auth.sh | 41 +- services/nginx/Dockerfile | 28 +- services/nginx/config/nginx.conf.template | 224 ++--- services/nginx/docker-entrypoint.sh | 50 ++ services/nginx/healthcheck.sh | 9 + 23 files changed, 1181 insertions(+), 845 deletions(-) create mode 100644 docs/deploy/nifi-runtime.md create mode 100644 nifi/docker/healthcheck.sh create mode 100644 scripts/tests/test_nifi_runtime.py create mode 100644 services/nginx/docker-entrypoint.sh create mode 100644 services/nginx/healthcheck.sh diff --git a/.dockerignore b/.dockerignore index 0386138a3..e997424a3 100644 --- a/.dockerignore +++ b/.dockerignore @@ -11,6 +11,10 @@ # endpoints, identifiers, or other data and are not reproducible inputs. nifi/conf/flow.json.gz nifi/conf/archive/ +nifi/user_scripts/logs/ +nifi/user_scripts/db/ +nifi/user_scripts/extensions/ +nifi/**/.venv/ # Development and operating-system artefacts within the included source tree. nifi/**/__pycache__/ diff --git a/.github/workflows/compose_config.yml b/.github/workflows/compose_config.yml index a8df8a9b6..68c3efb26 100644 --- a/.github/workflows/compose_config.yml +++ b/.github/workflows/compose_config.yml @@ -12,6 +12,7 @@ on: - "scripts/opensearch-ai-init.sh" - "scripts/tests/check_nifi_resources.py" - "scripts/tests/test_configuration.py" + - "scripts/tests/test_nifi_runtime.py" - "scripts/tests/check_opensearch_settings.py" - "scripts/tests/test_opensearch_deployment.py" - "services/elasticsearch/**" @@ -29,6 +30,7 @@ on: - "deploy/charts/**" - "deploy/helm/**" - "nifi/Dockerfile" + - "nifi/docker/**" - "services/nginx/Dockerfile" - "services/ollama-openai-proxy/Dockerfile" - "scripts/tests/check_deployment_images.py" @@ -45,6 +47,7 @@ on: - "scripts/opensearch-ai-init.sh" - "scripts/tests/check_nifi_resources.py" - "scripts/tests/test_configuration.py" + - "scripts/tests/test_nifi_runtime.py" - "scripts/tests/check_opensearch_settings.py" - "scripts/tests/test_opensearch_deployment.py" - "services/elasticsearch/**" @@ -62,6 +65,7 @@ on: - "deploy/charts/**" - "deploy/helm/**" - "nifi/Dockerfile" + - "nifi/docker/**" - "services/nginx/Dockerfile" - "services/ollama-openai-proxy/Dockerfile" - "scripts/tests/check_deployment_images.py" @@ -117,6 +121,9 @@ jobs: - name: Test configuration profiles and resource guards run: python3 -B -m unittest discover -s scripts/tests -p test_configuration.py -v + - name: Test NiFi upstream settings, persistence and credential isolation + run: python3 -B scripts/tests/test_nifi_runtime.py -v + - name: Test database bootstrap and credential isolation run: | python3 -B -m unittest discover -s scripts/tests -p 'test_*initialization.py' -v diff --git a/.github/workflows/smoke_nifi_services.yml b/.github/workflows/smoke_nifi_services.yml index 0e7e138a4..f278ee9d5 100644 --- a/.github/workflows/smoke_nifi_services.yml +++ b/.github/workflows/smoke_nifi_services.yml @@ -18,9 +18,11 @@ on: - "pyproject.toml" - "uv.lock" - "services/nginx/**" + - "deploy/services.yml" - "deploy/services.dev.yml" - "scripts/tests/check_nifi_resources.py" - "scripts/tests/smoke_nifi_services.sh" + - "scripts/tests/test_nifi_runtime.py" - "scripts/tests/smoke_http_checks.sh" - "!**/*.md" - "!**/README*" @@ -54,6 +56,12 @@ jobs: set -euo pipefail make -C deploy init-security-nifi + - name: Validate published nginx startup + run: | + set -euo pipefail + source deploy/export_env_vars.sh + docker compose -f deploy/services.yml run --rm --no-deps nifi-nginx nginx -t + - name: Smoke tests run: | set -euo pipefail @@ -64,7 +72,6 @@ jobs: run: | set -euo pipefail source deploy/export_env_vars.sh - set -euo pipefail docker compose -f deploy/services.dev.yml ps docker compose -f deploy/services.dev.yml logs --no-color nifi nifi-nginx @@ -73,5 +80,4 @@ jobs: run: | set -euo pipefail source deploy/export_env_vars.sh - set -euo pipefail docker compose -f deploy/services.dev.yml down -v diff --git a/deploy/Makefile b/deploy/Makefile index d384b77cb..7df05fcc1 100644 --- a/deploy/Makefile +++ b/deploy/Makefile @@ -351,8 +351,8 @@ build-nifi-image: ## Build the local NiFi image without starting services $(WITH_ENV) docker build \ --build-arg NIFI_VERSION="$${NIFI_VERSION:-2.12.0}" \ --build-arg NIFI_BASE_IMAGE="$${NIFI_BASE_IMAGE}" \ - --build-arg NIFI_UID="$${NIFI_UID:-1000}" \ - --build-arg NIFI_GID="$${NIFI_GID:-1000}" \ + --build-arg NIFI_UID="$${COGSTACK_LOCAL_UID:-1000}" \ + --build-arg NIFI_GID="$${COGSTACK_LOCAL_GID:-1000}" \ --tag "$${NIFI_LOCAL_DOCKER_IMAGE:-cogstack/cogstack-nifi:local}" \ --file ../nifi/Dockerfile \ .. diff --git a/deploy/export_env_vars.sh b/deploy/export_env_vars.sh index 40c0a8aa0..15a4034c1 100755 --- a/deploy/export_env_vars.sh +++ b/deploy/export_env_vars.sh @@ -75,6 +75,12 @@ _cogstack_load_environment() { } fi done + # Local image builds match bind-mounted configuration/certificate ownership. + # Published images retain their Dockerfile's default non-root UID/GID. + export COGSTACK_LOCAL_UID="${COGSTACK_LOCAL_UID:-$(id -u)}" + export COGSTACK_LOCAL_GID="${COGSTACK_LOCAL_GID:-$(id -g)}" + if [[ "$COGSTACK_LOCAL_UID" == 0 ]]; then export COGSTACK_LOCAL_UID=1000; fi + if [[ "$COGSTACK_LOCAL_GID" == 0 ]]; then export COGSTACK_LOCAL_GID=1000; fi export DOLLAR='$' if [[ "$restore_allexport" == 1 ]]; then set +a; fi echo "Configuration loaded from ${COGSTACK_CONFIG_DIR:-repository defaults}." diff --git a/deploy/nginx.env b/deploy/nginx.env index 214092e51..08cf535db 100644 --- a/deploy/nginx.env +++ b/deploy/nginx.env @@ -1,6 +1,3 @@ -# Published nginx proxy image, pinned to its multi-platform registry manifest. -NIFI_NGINX_DOCKER_IMAGE=cogstacksystems/nifi-nginx@sha256:486c933b05e14b4a3a1e4f966d832b27b2097f6789d8c9fbeb0c129d97a7136f - NGINX_KIBANA_HOST=kibana NGINX_KIBANA_PROXY_PORT=5601 NGINX_ES_NODE_SOURCE_INSTANCE_NAME="elasticsearch-1" @@ -9,6 +6,16 @@ NGINX_NIFI_SERVER_NAME=_ NGINX_ENVSUBST_OUTPUT_DIR=/etc/nginx +# Image startup supplies defaults when these nginx settings are omitted. +NGINX_PROXY_CONTEXT_PATH="" +NGINX_PROXY_SCHEME=https +NGINX_TLS_CERTIFICATE=/certificates/nifi/nifi.pem +NGINX_TLS_CERTIFICATE_KEY=/certificates/nifi/nifi.key +NGINX_UPSTREAM_CA=/certificates/root/root-ca.pem +NGINX_HEALTH_PORT=8080 +NGINX_WORKER_PROCESSES=1 +NGINX_RUNTIME_DIR=/tmp/nginx + NGINX_SHM_SIZE=512mb NGINX_DOCKER_CPU_MIN=1 diff --git a/deploy/nifi.env b/deploy/nifi.env index c06995f8c..04c8cbe0c 100644 --- a/deploy/nifi.env +++ b/deploy/nifi.env @@ -26,6 +26,9 @@ NIFI_VERSION="2.12.0" # Published NiFi 2.12.0 image, pinned to its multi-platform registry manifest. NIFI_DOCKER_IMAGE="cogstacksystems/cogstack-nifi@sha256:f3f0ce9c23f9e503042f646675ddef372acda5cffc2491f07a38996a95a948dd" +# Published nginx proxy image, pinned to its multi-platform registry manifest. +NIFI_NGINX_DOCKER_IMAGE=cogstacksystems/nifi-nginx@sha256:486c933b05e14b4a3a1e4f966d832b27b2097f6789d8c9fbeb0c129d97a7136f + # Local builds use a separate tag; never tag a build with the published digest. NIFI_LOCAL_DOCKER_IMAGE="cogstack/cogstack-nifi:local" NIFI_BASE_IMAGE="apache/nifi:2.12.0@sha256:6f7f2840dbf178c6dd3fa13435b9b358f3be3c6fb7417de467a35a267f4d28ad" @@ -66,6 +69,11 @@ NIFI_INTERNAL_PORT=8443 NIFI_EXTERNAL_PORT_NGINX=8443 NIFI_INTERNAL_PORT_NGINX=8443 +# NiFi endpoint used by nginx; the TLS name must match the NiFi certificate. +NIFI_UPSTREAM_HOST=nifi +NIFI_UPSTREAM_PORT="${NIFI_INTERNAL_PORT:-8443}" +NIFI_UPSTREAM_TLS_NAME=nifi + NIFI_OUTPUT_PORT=8082 NIFI_INPUT_SOCKET_PORT=10000 @@ -73,10 +81,10 @@ NIFI_UID=1000 NIFI_GID=1000 # env vars defining locations of proposed output logs for scripts -NIFI_USER_SCRIPT_LOGS_DIR="/opt/nifi/user_scripts/logs/" +NIFI_USER_SCRIPT_LOGS_DIR="/opt/nifi/nifi-current/logs/user_scripts/" # env vars for defining SQLITE DB locations -NIFI_USER_SCRIPT_DB_DIR="/opt/nifi/user_scripts/db/" +NIFI_USER_SCRIPT_DB_DIR="/opt/nifi/nifi-current/state/user_scripts/db/" # nifi.python.framework.source.directory=/opt/nifi/nifi-current/python/framework @@ -85,13 +93,16 @@ NIFI_PYTHON_FRAMEWORK_SOURCE_DIRECTORY="/opt/nifi/nifi-current/python/framework" # nifi.python.extensions.source.directory.default=/opt/nifi/nifi-current/python_extensions NIFI_PYTHON_EXTENSIONS_SOURCE_DIRECTORY_DEFAULT="/opt/nifi/nifi-current/python_extensions" -# nifi.python.working.directory=/opt/nifi/user_scripts -NIFI_PYTHON_WORKING_DIRECTORY="/opt/nifi/user_scripts" +# Generated Python environments/caches are separate from editable processor source. +NIFI_PYTHON_WORKING_DIRECTORY="/opt/nifi/nifi-current/work/python" NIFI_PYTHONPATH=$NIFI_PYTHON_FRAMEWORK_SOURCE_DIRECTORY NIFI_LOG_LEVEL="ERROR" -NIFI_AUTH=tls +# Apache startup's AUTH and provider settings are passed explicitly in Compose. +# Certificate reload and shutdown periods are configured in nifi.properties/bootstrap.conf. +NIFI_HEALTHCHECK_HOST=localhost +NIFI_HEALTHCHECK_CA_FILE="/security/certificates/root/root-ca.pem" NIFI_KEYSTORE_PATH="/security/certificates/nifi/nifi-keystore.jks" NIFI_TRUSTSTORE_PATH="/security/certificates/nifi/nifi-truststore.jks" diff --git a/deploy/services.dev.yml b/deploy/services.dev.yml index 6804f3828..dbe5e4fc4 100644 --- a/deploy/services.dev.yml +++ b/deploy/services.dev.yml @@ -8,38 +8,140 @@ x-nifi-logging-common: &nifi-logging-common max-size: ${NIFI_DOCKER_LOG_SIZE_PER_FILE:-250m} max-file: ${NIFI_DOCKER_LOG_NUM_FILES:-10} -x-all-env: &all-env - - "${COGSTACK_PROJECT_ENV_FILE:-./project.env}" - - "${COGSTACK_GENERAL_ENV_FILE:-./general.env}" - - "${COGSTACK_NIFI_ENV_FILE:-./nifi.env}" - - "${COGSTACK_GITEA_ENV_FILE:-./gitea.env}" - - "${COGSTACK_NGINX_ENV_FILE:-./nginx.env}" - - "${COGSTACK_TELEMETRY_ENV_FILE:-./telemetry.env}" - - "${COGSTACK_ELASTICSEARCH_ENV_FILE:-./elasticsearch.env}" - - "${COGSTACK_NETWORK_SETTINGS_ENV_FILE:-./network_settings.env}" - - "${COGSTACK_USERS_NIFI_ENV_FILE:-../security/env/users_nifi.env}" - - "${COGSTACK_USERS_NGINX_ENV_FILE:-../security/env/users_nginx.env}" - - "${COGSTACK_USERS_ELASTICSEARCH_ENV_FILE:-../security/env/users_elasticsearch.env}" - - "${COGSTACK_CERTIFICATES_GENERAL_ENV_FILE:-../security/env/certificates_general.env}" - - "${COGSTACK_CERTIFICATES_ELASTICSEARCH_ENV_FILE:-../security/env/certificates_elasticsearch.env}" - - "${COGSTACK_CERTIFICATES_NIFI_ENV_FILE:-../security/env/certificates_nifi.env}" - -x-es-env: &es-env - - "${COGSTACK_NETWORK_SETTINGS_ENV_FILE:-./network_settings.env}" - - "${COGSTACK_ELASTICSEARCH_ENV_FILE:-./elasticsearch.env}" - - "${COGSTACK_USERS_ELASTICSEARCH_ENV_FILE:-../security/env/users_elasticsearch.env}" - - "${COGSTACK_CERTIFICATES_ELASTICSEARCH_ENV_FILE:-../security/env/certificates_elasticsearch.env}" - -x-nifi-db-environment: &nifi-db-environment - POSTGRES_SAMPLES_DB_NAME: ${POSTGRES_SAMPLES_DB_NAME:-db_samples} - POSTGRES_SAMPLES_JDBC_URL: ${POSTGRES_SAMPLES_JDBC_URL:-jdbc:postgresql://samples-db:5432/${POSTGRES_SAMPLES_DB_NAME:-db_samples}?sslmode=verify-full&sslrootcert=/run/database-cas/samples-ca.crt} - POSTGRES_INGEST_USER_SAMPLES: ${POSTGRES_INGEST_USER_SAMPLES:-cogstack_samples} - POSTGRES_INGEST_PASSWORD_SAMPLES: ${POSTGRES_INGEST_PASSWORD_SAMPLES:?Set the sample ingestion password} - DATABASE_DB_NAME: ${DATABASE_DB_NAME:-cogstack} - DATABASE_INGEST_USER: ${DATABASE_INGEST_USER:-cogstack_ingest} - DATABASE_INGEST_PASSWORD: ${DATABASE_INGEST_PASSWORD:?Set the PostgreSQL ingestion password} - MSSQL_INGEST_USER: ${MSSQL_INGEST_USER:-cogstack_ingest} - MSSQL_INGEST_PASSWORD: ${MSSQL_INGEST_PASSWORD:?Set the MSSQL ingestion password} +x-env-files: + all: &all-env + - "${COGSTACK_PROJECT_ENV_FILE:-./project.env}" + - "${COGSTACK_GENERAL_ENV_FILE:-./general.env}" + - "${COGSTACK_NIFI_ENV_FILE:-./nifi.env}" + - "${COGSTACK_GITEA_ENV_FILE:-./gitea.env}" + - "${COGSTACK_NGINX_ENV_FILE:-./nginx.env}" + - "${COGSTACK_TELEMETRY_ENV_FILE:-./telemetry.env}" + - "${COGSTACK_ELASTICSEARCH_ENV_FILE:-./elasticsearch.env}" + - "${COGSTACK_NETWORK_SETTINGS_ENV_FILE:-./network_settings.env}" + - "${COGSTACK_USERS_NIFI_ENV_FILE:-../security/env/users_nifi.env}" + - "${COGSTACK_USERS_NGINX_ENV_FILE:-../security/env/users_nginx.env}" + - "${COGSTACK_USERS_ELASTICSEARCH_ENV_FILE:-../security/env/users_elasticsearch.env}" + - "${COGSTACK_CERTIFICATES_GENERAL_ENV_FILE:-../security/env/certificates_general.env}" + - "${COGSTACK_CERTIFICATES_ELASTICSEARCH_ENV_FILE:-../security/env/certificates_elasticsearch.env}" + - "${COGSTACK_CERTIFICATES_NIFI_ENV_FILE:-../security/env/certificates_nifi.env}" + + search: &es-env + - "${COGSTACK_NETWORK_SETTINGS_ENV_FILE:-./network_settings.env}" + - "${COGSTACK_ELASTICSEARCH_ENV_FILE:-./elasticsearch.env}" + - "${COGSTACK_USERS_ELASTICSEARCH_ENV_FILE:-../security/env/users_elasticsearch.env}" + - "${COGSTACK_CERTIFICATES_ELASTICSEARCH_ENV_FILE:-../security/env/certificates_elasticsearch.env}" + + nifi-nginx: &nifi-nginx-env-files + - "${COGSTACK_NGINX_ENV_FILE:-./nginx.env}" + +x-env: + nifi: &nifi-env + POSTGRES_SAMPLES_DB_NAME: ${POSTGRES_SAMPLES_DB_NAME:-db_samples} + POSTGRES_SAMPLES_JDBC_URL: ${POSTGRES_SAMPLES_JDBC_URL:-jdbc:postgresql://samples-db:5432/${POSTGRES_SAMPLES_DB_NAME:-db_samples}?sslmode=verify-full&sslrootcert=/run/database-cas/samples-ca.crt} + POSTGRES_INGEST_USER_SAMPLES: ${POSTGRES_INGEST_USER_SAMPLES:-cogstack_samples} + POSTGRES_INGEST_PASSWORD_SAMPLES: ${POSTGRES_INGEST_PASSWORD_SAMPLES:?Set the sample ingestion password} + DATABASE_DB_NAME: ${DATABASE_DB_NAME:-cogstack} + DATABASE_INGEST_USER: ${DATABASE_INGEST_USER:-cogstack_ingest} + DATABASE_INGEST_PASSWORD: ${DATABASE_INGEST_PASSWORD:?Set the PostgreSQL ingestion password} + MSSQL_INGEST_USER: ${MSSQL_INGEST_USER:-cogstack_ingest} + MSSQL_INGEST_PASSWORD: ${MSSQL_INGEST_PASSWORD:?Set the MSSQL ingestion password} + + # Apache's native startup settings; existing deployment env names remain supported. + AUTH: ${AUTH:-tls} + NIFI_SECURITY_USER_AUTHORIZER: ${NIFI_SECURITY_USER_AUTHORIZER:-single-user-authorizer} + NIFI_SECURITY_USER_LOGIN_IDENTITY_PROVIDER: ${NIFI_SECURITY_USER_LOGIN_IDENTITY_PROVIDER:-single-user-provider} + SINGLE_USER_CREDENTIALS_USERNAME: ${SINGLE_USER_CREDENTIALS_USERNAME:-${NIFI_SINGLE_USER_CREDENTIALS_USERNAME}} + SINGLE_USER_CREDENTIALS_PASSWORD: ${SINGLE_USER_CREDENTIALS_PASSWORD:-${NIFI_SINGLE_USER_CREDENTIALS_PASSWORD}} + KEYSTORE_PATH: ${KEYSTORE_PATH:-${NIFI_KEYSTORE_PATH}} + KEYSTORE_TYPE: ${KEYSTORE_TYPE:-${NIFI_KEYSTORE_TYPE}} + KEYSTORE_PASSWORD: ${KEYSTORE_PASSWORD:-${NIFI_KEYSTORE_PASSWORD}} + TRUSTSTORE_PATH: ${TRUSTSTORE_PATH:-${NIFI_TRUSTSTORE_PATH}} + TRUSTSTORE_TYPE: ${TRUSTSTORE_TYPE:-${NIFI_TRUSTSTORE_TYPE}} + TRUSTSTORE_PASSWORD: ${TRUSTSTORE_PASSWORD:-${NIFI_TRUSTSTORE_PASSWORD}} + NIFI_WEB_HTTPS_PORT: ${NIFI_WEB_HTTPS_PORT:-${NIFI_INTERNAL_PORT:-8443}} + NIFI_REMOTE_INPUT_SOCKET_PORT: ${NIFI_REMOTE_INPUT_SOCKET_PORT:-${NIFI_INPUT_SOCKET_PORT:-10000}} + PYTHONPATH: ${NIFI_PYTHONPATH:-/opt/nifi/nifi-current/python/framework} + JVM_OPTS: ${NIFI_JVM_OPTS:--XX:+UseG1GC -XX:MaxGCPauseMillis=200 -XX:+ParallelRefProcEnabled -Djava.security.egd=file:/dev/./urandom} + + nifi-nginx: &nifi-nginx-env + NIFI_INTERNAL_PORT_NGINX: ${NIFI_INTERNAL_PORT_NGINX:-8443} + NIFI_WEB_PROXY_PORT: ${NIFI_WEB_PROXY_PORT:-8443} + NIFI_INTERNAL_PORT: ${NIFI_INTERNAL_PORT:-8443} + NIFI_UPSTREAM_HOST: ${NIFI_UPSTREAM_HOST:-nifi} + NIFI_UPSTREAM_PORT: ${NIFI_UPSTREAM_PORT:-${NIFI_INTERNAL_PORT:-8443}} + NIFI_UPSTREAM_TLS_NAME: ${NIFI_UPSTREAM_TLS_NAME:-nifi} + +x-mounts: + nifi: &nifi-volumes + # Drivers + - ../nifi/drivers:/opt/nifi/drivers:ro + + # User overrides bundled in the image + - ../nifi/user_scripts:/opt/nifi/user_scripts:rw + - ../nifi/user_schemas:/opt/nifi/user_schemas:ro + + # Python processors (NiFi 2.x) + - ../nifi/user_python_extensions:/opt/nifi/nifi-current/python_extensions:rw + + # Security certificates + - ../security/certificates/nifi:/security/certificates/nifi:ro + - ../security/certificates/elastic/opensearch/elasticsearch:/security/certificates/elastic/opensearch/elasticsearch:ro + - ../security/certificates/elastic/opensearch/elastic-stack-ca.crt.pem:/security/certificates/elastic/opensearch/elastic-stack-ca.crt.pem:ro + - ../security/certificates/elastic/opensearch/es_kibana_client.pem:/security/certificates/elastic/opensearch/es_kibana_client.pem:ro + - ../security/certificates/elastic/opensearch/es_kibana_client.key:/security/certificates/elastic/opensearch/es_kibana_client.key:ro + - ../security/certificates/root/root-ca.pem:/security/certificates/root/root-ca.pem:ro + # Database clients receive only the public CA from each TLS bundle. + - type: bind + source: ${POSTGRES_SAMPLES_TLS_DIR:-../security/certificates/postgresql/samples}/ca.crt + target: /run/database-cas/samples-ca.crt + read_only: true + bind: + create_host_path: false + - type: bind + source: ${POSTGRES_DATABANK_TLS_DIR:-../security/certificates/postgresql/databank}/ca.crt + target: /run/database-cas/databank-ca.crt + read_only: true + bind: + create_host_path: false + + # Apache startup configures image defaults in this persistent, writable directory. + - nifi-vol-conf:/opt/nifi/nifi-current/conf + + # Ingest data directory + - ${NIFI_DATA_PATH:-../data/}:/data/:rw + + # MedCAT models + - ${NIFI_MEDCAT_SERVICE_MODEL_PRODUCTION_PATH:-../services/cogstack-nlp/medcat-service/models/}:/opt/models:rw + + # NiFi repositories/state + - nifi-vol-logs:/opt/nifi/nifi-current/logs + - nifi-vol-provenance:/opt/nifi/nifi-current/provenance_repository + - nifi-vol-database:/opt/nifi/nifi-current/database_repository + - nifi-vol-flowfiles:/opt/nifi/nifi-current/flowfile_repository + - nifi-vol-content:/opt/nifi/nifi-current/content_repository + - nifi-vol-state:/opt/nifi/nifi-current/state + + # Flowfile error output + - nifi-vol-errors:/opt/nifi/pipeline/flowfile-errors + + nifi-nginx-tmpfs: &nifi-nginx-tmpfs + - /tmp:mode=1777 + + nifi-nginx: &nifi-nginx-volumes + - ../services/nginx/sites-enabled:/etc/nginx/sites-enabled:ro + - ../security/certificates/nifi:/certificates/nifi:ro + - ../security/certificates/root/root-ca.pem:/certificates/root/root-ca.pem:ro + + otel-collector: &otel-collector-volumes + - ./observability/otel-collector.yaml:/etc/otelcol-contrib/config.yaml:ro + + data-prepper: &data-prepper-volumes + - ./observability/data-prepper/config/data-prepper-config.yaml:/usr/share/data-prepper/config/data-prepper-config.yaml:ro + - ./observability/data-prepper/pipelines:/usr/share/data-prepper/pipelines:ro + +x-networks: + cognet: &cognet + - cognet x-common-hosts: &common-hosts - ${ELASTICSEARCH_1_HOST_NAME:-test-1:0.0.0.0} @@ -63,64 +165,7 @@ x-nifi-common: &nifi-common restart: unless-stopped env_file: *all-env extra_hosts: *common-hosts - networks: - - cognet - -x-nifi-volumes: &nifi-volumes - # Drivers - - ../nifi/drivers:/opt/nifi/drivers:ro - - # User overrides bundled in the image - - ../nifi/user_scripts:/opt/nifi/user_scripts:rw - - ../nifi/user_schemas:/opt/nifi/user_schemas:ro - - # Python processors (NiFi 2.x) - - ../nifi/user_python_extensions:/opt/nifi/nifi-current/python_extensions:rw - - # Security certificates - - ../security/certificates/nifi:/security/certificates/nifi:ro - - ../security/certificates/elastic/opensearch/elasticsearch:/security/certificates/elastic/opensearch/elasticsearch:ro - - ../security/certificates/elastic/opensearch/elastic-stack-ca.crt.pem:/security/certificates/elastic/opensearch/elastic-stack-ca.crt.pem:ro - - ../security/certificates/elastic/opensearch/es_kibana_client.pem:/security/certificates/elastic/opensearch/es_kibana_client.pem:ro - - ../security/certificates/elastic/opensearch/es_kibana_client.key:/security/certificates/elastic/opensearch/es_kibana_client.key:ro - - ../security/certificates/root/root-ca.pem:/security/certificates/root/root-ca.pem:ro - # Database clients receive only the public CA from each TLS bundle. - - type: bind - source: ${POSTGRES_SAMPLES_TLS_DIR:-../security/certificates/postgresql/samples}/ca.crt - target: /run/database-cas/samples-ca.crt - read_only: true - bind: - create_host_path: false - - type: bind - source: ${POSTGRES_DATABANK_TLS_DIR:-../security/certificates/postgresql/databank}/ca.crt - target: /run/database-cas/databank-ca.crt - read_only: true - bind: - create_host_path: false - - # Security helper scripts - - ../security/scripts/nifi_create_single_user_auth.sh:/opt/nifi/nifi-current/security_scripts/nifi_create_single_user_auth.sh:ro - - # NiFi configuration - - ../nifi/conf/:/opt/nifi/nifi-current/conf/:rw - - # Ingest data directory - - ${NIFI_DATA_PATH:-../data/}:/data/:rw - - # MedCAT models - - ${NIFI_MEDCAT_SERVICE_MODEL_PRODUCTION_PATH:-../services/cogstack-nlp/medcat-service/models/}:/opt/models:rw - - # NiFi repositories/state - - nifi-vol-logs:/opt/nifi/nifi-current/logs - - nifi-vol-provenance:/opt/nifi/nifi-current/provenance_repository - - nifi-vol-database:/opt/nifi/nifi-current/database_repository - - nifi-vol-flowfiles:/opt/nifi/nifi-current/flowfile_repository - - nifi-vol-content:/opt/nifi/nifi-current/content_repository - - nifi-vol-state:/opt/nifi/nifi-current/state - - # Flowfile error output - - nifi-vol-errors:/opt/nifi/pipeline/flowfile-errors - + networks: *cognet #---------------------------------------------------------------------------# # Used services # #---------------------------------------------------------------------------# @@ -139,13 +184,12 @@ services: args: NIFI_VERSION: ${NIFI_VERSION:-2.12.0} NIFI_BASE_IMAGE: ${NIFI_BASE_IMAGE:?Source deploy/export_env_vars.sh first} + NIFI_UID: ${COGSTACK_LOCAL_UID:-1000} + NIFI_GID: ${COGSTACK_LOCAL_GID:-1000} container_name: cogstack-nifi hostname: nifi shm_size: ${NIFI_DOCKER_SHM_SIZE:-"1g"} - environment: - <<: *nifi-db-environment - PYTHONPATH: ${NIFI_PYTHONPATH:-/opt/nifi/nifi-current/python/framework} - JVM_OPTS: ${NIFI_JVM_OPTS:--XX:+UseG1GC -XX:MaxGCPauseMillis=200 -XX:+ParallelRefProcEnabled -Djava.security.egd=file:/dev/./urandom} + environment: *nifi-env deploy: resources: limits: @@ -156,11 +200,7 @@ services: memory: "${NIFI_DOCKER_RAM}" volumes: *nifi-volumes - # INFO : Uncomment the below line to generate your own USERNAME and PASSWORD, - # a bit messy this way as you will need to copy the credentials back - # to the "login-identity-providers.xml" section. - # entrypoint: bash -c "/opt/nifi/nifi-current/bin/nifi.sh set-single-user-credentials admin admincogstacknifi" - tty: true + stop_grace_period: 120s ports: - "${NIFI_OUTPUT_PORT:-8082}:${NIFI_INTERNAL_PORT:-8443}" - "${NIFI_INPUT_SOCKET_PORT:-10000}" @@ -171,10 +211,17 @@ services: build: context: ../services/nginx dockerfile: Dockerfile + args: + NGINX_UID: ${COGSTACK_LOCAL_UID:-1000} + NGINX_GID: ${COGSTACK_LOCAL_GID:-1000} container_name: cogstack-nifi-nginx restart: always shm_size: ${NGINX_SHM_SIZE:-512mb} - env_file: *all-env + env_file: *nifi-nginx-env-files + environment: *nifi-nginx-env + stop_grace_period: 90s + read_only: true + tmpfs: *nifi-nginx-tmpfs deploy: resources: limits: @@ -183,15 +230,10 @@ services: reservations: cpus: "${NGINX_DOCKER_CPU_MIN}" memory: "${NGINX_DOCKER_RAM}" - volumes: - - ../services/nginx/sites-enabled:/etc/nginx/sites-enabled:ro - - ../services/nginx/config/nginx.conf.template:/etc/nginx/templates/nginx.conf.template:ro - - ../security/certificates/nifi:/certificates/nifi:ro - - ../security/certificates/root/root-ca.pem:/certificates/root/root-ca.pem:ro + volumes: *nifi-nginx-volumes ports: - "${NIFI_EXTERNAL_PORT_NGINX:-8443}:${NIFI_INTERNAL_PORT_NGINX:-8443}" - networks: - - cognet + networks: *cognet extra_hosts: *common-hosts logging: *nifi-logging-common @@ -204,14 +246,12 @@ services: restart: unless-stopped command: ["--config=/etc/otelcol-contrib/config.yaml"] env_file: *all-env - volumes: - - ./observability/otel-collector.yaml:/etc/otelcol-contrib/config.yaml:ro + volumes: *otel-collector-volumes ports: - "${OTEL_COLLECTOR_OTLP_GRPC_PORT:-4317}:4317" - "${OTEL_COLLECTOR_OTLP_HTTP_PORT:-4318}:4318" - "${OTEL_COLLECTOR_HEALTH_PORT:-13133}:13133" - networks: - - cognet + networks: *cognet extra_hosts: *common-hosts logging: *nifi-logging-common @@ -220,14 +260,11 @@ services: container_name: cogstack-data-prepper restart: unless-stopped env_file: *all-env - volumes: - - ./observability/data-prepper/config/data-prepper-config.yaml:/usr/share/data-prepper/config/data-prepper-config.yaml:ro - - ./observability/data-prepper/pipelines:/usr/share/data-prepper/pipelines:ro + volumes: *data-prepper-volumes ports: - "${DATA_PREPPER_API_PORT:-4900}:4900" - "${DATA_PREPPER_OTEL_TRACE_PORT:-21890}:21890" - networks: - - cognet + networks: *cognet extra_hosts: *common-hosts logging: *nifi-logging-common @@ -235,6 +272,7 @@ services: # Docker named volumes # #---------------------------------------------------------------------------# volumes: + nifi-vol-conf: # NiFi related nifi-vol-logs: driver: local diff --git a/deploy/services.yml b/deploy/services.yml index ad442b49b..8d2c451d4 100644 --- a/deploy/services.yml +++ b/deploy/services.yml @@ -19,38 +19,397 @@ x-logging-common: &logging-common max-size: ${DOCKER_LOG_SIZE_PER_FILE:-100m} max-file: ${DOCKER_LOG_NUM_FILES:-10} -x-all-env: &all-env - - "${COGSTACK_PROJECT_ENV_FILE:-./project.env}" - - "${COGSTACK_GENERAL_ENV_FILE:-./general.env}" - - "${COGSTACK_NIFI_ENV_FILE:-./nifi.env}" - - "${COGSTACK_GITEA_ENV_FILE:-./gitea.env}" - - "${COGSTACK_NGINX_ENV_FILE:-./nginx.env}" - - "${COGSTACK_TELEMETRY_ENV_FILE:-./telemetry.env}" - - "${COGSTACK_ELASTICSEARCH_ENV_FILE:-./elasticsearch.env}" - - "${COGSTACK_NETWORK_SETTINGS_ENV_FILE:-./network_settings.env}" - - "${COGSTACK_USERS_NIFI_ENV_FILE:-../security/env/users_nifi.env}" - - "${COGSTACK_USERS_NGINX_ENV_FILE:-../security/env/users_nginx.env}" - - "${COGSTACK_USERS_ELASTICSEARCH_ENV_FILE:-../security/env/users_elasticsearch.env}" - - "${COGSTACK_CERTIFICATES_GENERAL_ENV_FILE:-../security/env/certificates_general.env}" - - "${COGSTACK_CERTIFICATES_ELASTICSEARCH_ENV_FILE:-../security/env/certificates_elasticsearch.env}" - - "${COGSTACK_CERTIFICATES_NIFI_ENV_FILE:-../security/env/certificates_nifi.env}" - -x-es-env: &es-env - - "${COGSTACK_NETWORK_SETTINGS_ENV_FILE:-./network_settings.env}" - - "${COGSTACK_ELASTICSEARCH_ENV_FILE:-./elasticsearch.env}" - - "${COGSTACK_USERS_ELASTICSEARCH_ENV_FILE:-../security/env/users_elasticsearch.env}" - - "${COGSTACK_CERTIFICATES_ELASTICSEARCH_ENV_FILE:-../security/env/certificates_elasticsearch.env}" - -x-nifi-db-environment: &nifi-db-environment - POSTGRES_SAMPLES_DB_NAME: ${POSTGRES_SAMPLES_DB_NAME:-db_samples} - POSTGRES_SAMPLES_JDBC_URL: ${POSTGRES_SAMPLES_JDBC_URL:-jdbc:postgresql://samples-db:5432/${POSTGRES_SAMPLES_DB_NAME:-db_samples}?sslmode=verify-full&sslrootcert=/run/database-cas/samples-ca.crt} - POSTGRES_INGEST_USER_SAMPLES: ${POSTGRES_INGEST_USER_SAMPLES:-cogstack_samples} - POSTGRES_INGEST_PASSWORD_SAMPLES: ${POSTGRES_INGEST_PASSWORD_SAMPLES:?Set the sample ingestion password} - DATABASE_DB_NAME: ${DATABASE_DB_NAME:-cogstack} - DATABASE_INGEST_USER: ${DATABASE_INGEST_USER:-cogstack_ingest} - DATABASE_INGEST_PASSWORD: ${DATABASE_INGEST_PASSWORD:?Set the PostgreSQL ingestion password} - MSSQL_INGEST_USER: ${MSSQL_INGEST_USER:-cogstack_ingest} - MSSQL_INGEST_PASSWORD: ${MSSQL_INGEST_PASSWORD:?Set the MSSQL ingestion password} +x-env-files: + all: &all-env + - "${COGSTACK_PROJECT_ENV_FILE:-./project.env}" + - "${COGSTACK_GENERAL_ENV_FILE:-./general.env}" + - "${COGSTACK_NIFI_ENV_FILE:-./nifi.env}" + - "${COGSTACK_GITEA_ENV_FILE:-./gitea.env}" + - "${COGSTACK_NGINX_ENV_FILE:-./nginx.env}" + - "${COGSTACK_TELEMETRY_ENV_FILE:-./telemetry.env}" + - "${COGSTACK_ELASTICSEARCH_ENV_FILE:-./elasticsearch.env}" + - "${COGSTACK_NETWORK_SETTINGS_ENV_FILE:-./network_settings.env}" + - "${COGSTACK_USERS_NIFI_ENV_FILE:-../security/env/users_nifi.env}" + - "${COGSTACK_USERS_NGINX_ENV_FILE:-../security/env/users_nginx.env}" + - "${COGSTACK_USERS_ELASTICSEARCH_ENV_FILE:-../security/env/users_elasticsearch.env}" + - "${COGSTACK_CERTIFICATES_GENERAL_ENV_FILE:-../security/env/certificates_general.env}" + - "${COGSTACK_CERTIFICATES_ELASTICSEARCH_ENV_FILE:-../security/env/certificates_elasticsearch.env}" + - "${COGSTACK_CERTIFICATES_NIFI_ENV_FILE:-../security/env/certificates_nifi.env}" + + search: &es-env + - "${COGSTACK_NETWORK_SETTINGS_ENV_FILE:-./network_settings.env}" + - "${COGSTACK_ELASTICSEARCH_ENV_FILE:-./elasticsearch.env}" + - "${COGSTACK_USERS_ELASTICSEARCH_ENV_FILE:-../security/env/users_elasticsearch.env}" + - "${COGSTACK_CERTIFICATES_ELASTICSEARCH_ENV_FILE:-../security/env/certificates_elasticsearch.env}" + + beats: &beats-env-files + - "${COGSTACK_ELASTICSEARCH_ENV_FILE:-./elasticsearch.env}" + - "${COGSTACK_USERS_ELASTICSEARCH_ENV_FILE:-../security/env/users_elasticsearch.env}" + + clickhouse: &clickhouse-env-files + - "${COGSTACK_DATABASE_ENV_FILE:-./database.env}" + + ollama: &ollama-env-files + - "${COGSTACK_OLLAMA_ENV_FILE:-./ollama.env}" + + open-webui: &open-webui-env-files + - "${COGSTACK_OPENWEBUI_ENV_FILE:-./openwebui.env}" + + nifi-nginx: &nifi-nginx-env-files + - "${COGSTACK_NGINX_ENV_FILE:-./nginx.env}" + +x-env: + nifi: &nifi-env + POSTGRES_SAMPLES_DB_NAME: ${POSTGRES_SAMPLES_DB_NAME:-db_samples} + POSTGRES_SAMPLES_JDBC_URL: ${POSTGRES_SAMPLES_JDBC_URL:-jdbc:postgresql://samples-db:5432/${POSTGRES_SAMPLES_DB_NAME:-db_samples}?sslmode=verify-full&sslrootcert=/run/database-cas/samples-ca.crt} + POSTGRES_INGEST_USER_SAMPLES: ${POSTGRES_INGEST_USER_SAMPLES:-cogstack_samples} + POSTGRES_INGEST_PASSWORD_SAMPLES: ${POSTGRES_INGEST_PASSWORD_SAMPLES:?Set the sample ingestion password} + DATABASE_DB_NAME: ${DATABASE_DB_NAME:-cogstack} + DATABASE_INGEST_USER: ${DATABASE_INGEST_USER:-cogstack_ingest} + DATABASE_INGEST_PASSWORD: ${DATABASE_INGEST_PASSWORD:?Set the PostgreSQL ingestion password} + MSSQL_INGEST_USER: ${MSSQL_INGEST_USER:-cogstack_ingest} + MSSQL_INGEST_PASSWORD: ${MSSQL_INGEST_PASSWORD:?Set the MSSQL ingestion password} + + # Apache's native startup settings; existing deployment env names remain supported. + AUTH: ${AUTH:-tls} + NIFI_SECURITY_USER_AUTHORIZER: ${NIFI_SECURITY_USER_AUTHORIZER:-single-user-authorizer} + NIFI_SECURITY_USER_LOGIN_IDENTITY_PROVIDER: ${NIFI_SECURITY_USER_LOGIN_IDENTITY_PROVIDER:-single-user-provider} + SINGLE_USER_CREDENTIALS_USERNAME: ${SINGLE_USER_CREDENTIALS_USERNAME:-${NIFI_SINGLE_USER_CREDENTIALS_USERNAME}} + SINGLE_USER_CREDENTIALS_PASSWORD: ${SINGLE_USER_CREDENTIALS_PASSWORD:-${NIFI_SINGLE_USER_CREDENTIALS_PASSWORD}} + KEYSTORE_PATH: ${KEYSTORE_PATH:-${NIFI_KEYSTORE_PATH}} + KEYSTORE_TYPE: ${KEYSTORE_TYPE:-${NIFI_KEYSTORE_TYPE}} + KEYSTORE_PASSWORD: ${KEYSTORE_PASSWORD:-${NIFI_KEYSTORE_PASSWORD}} + TRUSTSTORE_PATH: ${TRUSTSTORE_PATH:-${NIFI_TRUSTSTORE_PATH}} + TRUSTSTORE_TYPE: ${TRUSTSTORE_TYPE:-${NIFI_TRUSTSTORE_TYPE}} + TRUSTSTORE_PASSWORD: ${TRUSTSTORE_PASSWORD:-${NIFI_TRUSTSTORE_PASSWORD}} + NIFI_WEB_HTTPS_PORT: ${NIFI_WEB_HTTPS_PORT:-${NIFI_INTERNAL_PORT:-8443}} + NIFI_REMOTE_INPUT_SOCKET_PORT: ${NIFI_REMOTE_INPUT_SOCKET_PORT:-${NIFI_INPUT_SOCKET_PORT:-10000}} + PYTHONPATH: ${NIFI_PYTHONPATH:-/opt/nifi/nifi-current/python/framework} + JVM_OPTS: ${NIFI_JVM_OPTS:--XX:+UseG1GC -XX:MaxGCPauseMillis=200 -XX:+ParallelRefProcEnabled -Djava.security.egd=file:/dev/./urandom} + + nifi-nginx: &nifi-nginx-env + NIFI_INTERNAL_PORT_NGINX: ${NIFI_INTERNAL_PORT_NGINX:-8443} + NIFI_WEB_PROXY_PORT: ${NIFI_WEB_PROXY_PORT:-8443} + # Keep the published image's template renderer compatible until the next image release. + NIFI_UPSTREAM_HOST: ${NIFI_UPSTREAM_HOST:-nifi} + NIFI_UPSTREAM_PORT: ${NIFI_UPSTREAM_PORT:-${NIFI_INTERNAL_PORT:-8443}} + NIFI_UPSTREAM_TLS_NAME: ${NIFI_UPSTREAM_TLS_NAME:-nifi} + NGINX_NIFI_SERVER_NAME: ${NGINX_NIFI_SERVER_NAME:-_} + NGINX_PROXY_CONTEXT_PATH: ${NGINX_PROXY_CONTEXT_PATH:-} + NGINX_PROXY_SCHEME: ${NGINX_PROXY_SCHEME:-https} + NGINX_TLS_CERTIFICATE: ${NGINX_TLS_CERTIFICATE:-/certificates/nifi/nifi.pem} + NGINX_TLS_CERTIFICATE_KEY: ${NGINX_TLS_CERTIFICATE_KEY:-/certificates/nifi/nifi.key} + NGINX_UPSTREAM_CA: ${NGINX_UPSTREAM_CA:-/certificates/root/root-ca.pem} + NGINX_HEALTH_PORT: ${NGINX_HEALTH_PORT:-8080} + NGINX_WORKER_PROCESSES: ${NGINX_WORKER_PROCESSES:-1} + NGINX_RUNTIME_DIR: ${NGINX_RUNTIME_DIR:-/tmp/nginx} + NGINX_DNS_RESOLVER: ${NGINX_DNS_RESOLVER:-127.0.0.11} + + es-common: &es-common-env + ELASTICSEARCH_VERSION: ${ELASTICSEARCH_VERSION:-opensearch} + + samples-db: &samples-db-env + PGSSLMODE: verify-full + PGSSLROOTCERT: /run/postgresql/tls/ca.crt + POSTGRES_USER: ${POSTGRES_USER_SAMPLES:-test} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD_SAMPLES:?Set the sample bootstrap password} + POSTGRES_DB: ${POSTGRES_SAMPLES_DB_NAME:-db_samples} + COGSTACK_INGEST_USER: ${POSTGRES_INGEST_USER_SAMPLES:-cogstack_samples} + COGSTACK_INGEST_PASSWORD: ${POSTGRES_INGEST_PASSWORD_SAMPLES:?Set the sample ingestion password} + + cogstack-databank-db: &cogstack-databank-db-env + PGSSLMODE: verify-full + PGSSLROOTCERT: /run/postgresql/tls/ca.crt + POSTGRES_USER: ${DATABASE_USER:-admin} + POSTGRES_PASSWORD: ${DATABASE_PASSWORD:?Set the PostgreSQL bootstrap password} + POSTGRES_DB: ${DATABASE_DB_NAME:-cogstack} + POSTGRES_DB_SCHEMA_PREFIX: ${POSTGRES_DB_SCHEMA_PREFIX:-cogstack_db} + COGSTACK_INGEST_USER: ${DATABASE_INGEST_USER:-cogstack_ingest} + COGSTACK_INGEST_PASSWORD: ${DATABASE_INGEST_PASSWORD:?Set the PostgreSQL ingestion password} + + cogstack-databank-db-mssql: &cogstack-databank-db-mssql-env + ACCEPT_EULA: "Y" + MSSQL_SA_PASSWORD: ${MSSQL_SA_PASSWORD:?Set the SQL Server bootstrap password} + + cogstack-databank-db-mssql-init: &cogstack-databank-db-mssql-init-env + MSSQL_SERVER: cogstack-databank-db-mssql,1433 + MSSQL_SA_USER: ${MSSQL_SA_USER:-sa} + MSSQL_SA_PASSWORD: ${MSSQL_SA_PASSWORD:?Set the SQL Server bootstrap password} + DATABASE_DB_NAME: ${DATABASE_DB_NAME:-cogstack} + POSTGRES_DB_SCHEMA_PREFIX: ${POSTGRES_DB_SCHEMA_PREFIX:-cogstack_db} + COGSTACK_INGEST_USER: ${MSSQL_INGEST_USER:-cogstack_ingest} + COGSTACK_INGEST_PASSWORD: ${MSSQL_INGEST_PASSWORD:?Set the MSSQL ingestion password} + MSSQL_TRUST_SERVER_CERTIFICATE: ${MSSQL_TRUST_SERVER_CERTIFICATE:-true} + MSSQL_TLS_HOSTNAME: ${MSSQL_TLS_HOSTNAME:-} + + elasticsearch-1: &elasticsearch-1-env + node.name: ${ELASTICSEARCH_NODE_1_NAME:-es01} + + elasticsearch-2: &elasticsearch-2-env + node.name: ${ELASTICSEARCH_NODE_2_NAME:-es02} + + elasticsearch-3: &elasticsearch-3-env + node.name: ${ELASTICSEARCH_NODE_3_NAME:-es03} + + ollama-openai-proxy: &ollama-openai-proxy-env + OLLAMA_UPSTREAM_HOST: ollama + OLLAMA_UPSTREAM_PORT: 11434 + OLLAMA_UPSTREAM_TIMEOUT_SECONDS: ${OPENSEARCH_AI_READ_TIMEOUT_SECONDS:-360} + +x-mounts: + nifi: &nifi-volumes + # Drivers + - ../nifi/drivers:/opt/nifi/drivers:ro + + # User overrides bundled in the image + - ../nifi/user_scripts:/opt/nifi/user_scripts:rw + - ../nifi/user_schemas:/opt/nifi/user_schemas:rw + + # Live Python package imports + - ../nifi/user_scripts/:/opt/nifi/nifi-current/python/framework/nifi/user_scripts:rw + + # Python processors (NiFi 2.x) + - ../nifi/user_python_extensions:/opt/nifi/nifi-current/python_extensions:rw + + # Security certificates + - ../security/certificates/nifi:/security/certificates/nifi:ro + - ../security/certificates/elastic/opensearch/elasticsearch:/security/certificates/elastic/opensearch/elasticsearch:ro + - ../security/certificates/elastic/opensearch/elastic-stack-ca.crt.pem:/security/certificates/elastic/opensearch/elastic-stack-ca.crt.pem:ro + - ../security/certificates/elastic/opensearch/es_kibana_client.pem:/security/certificates/elastic/opensearch/es_kibana_client.pem:ro + - ../security/certificates/elastic/opensearch/es_kibana_client.key:/security/certificates/elastic/opensearch/es_kibana_client.key:ro + - ../security/certificates/root/root-ca.pem:/security/certificates/root/root-ca.pem:ro + # Database clients receive only the public CA from each TLS bundle. + - type: bind + source: ${POSTGRES_SAMPLES_TLS_DIR:-../security/certificates/postgresql/samples}/ca.crt + target: /run/database-cas/samples-ca.crt + read_only: true + bind: + create_host_path: false + - type: bind + source: ${POSTGRES_DATABANK_TLS_DIR:-../security/certificates/postgresql/databank}/ca.crt + target: /run/database-cas/databank-ca.crt + read_only: true + bind: + create_host_path: false + + # NiFi configuration + - ../nifi/conf/:/opt/nifi/nifi-current/conf/:rw + + # Ingest data directory + - ${NIFI_DATA_PATH:-../data/}:/data/:rw + + # MedCAT models + - ${NIFI_MEDCAT_SERVICE_MODEL_PRODUCTION_PATH:-../services/cogstack-nlp/medcat-service/models/}:/opt/models:rw + + # NiFi repositories/state + - nifi-vol-logs:/opt/nifi/nifi-current/logs + - nifi-vol-provenance:/opt/nifi/nifi-current/provenance_repository + - nifi-vol-database:/opt/nifi/nifi-current/database_repository + - nifi-vol-flowfiles:/opt/nifi/nifi-current/flowfile_repository + - nifi-vol-content:/opt/nifi/nifi-current/content_repository + - nifi-vol-state:/opt/nifi/nifi-current/state + + # Flowfile error output + - nifi-vol-errors:/opt/nifi/pipeline/flowfile-errors + + search-common: &es-common-volumes + # Shared configs + - ../services/elasticsearch/search-entrypoint.sh:/usr/local/bin/cogstack-search-entrypoint.sh:ro + - ../services/elasticsearch/config/${ELASTICSEARCH_VERSION:-opensearch}.yml:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/config-input/${ELASTICSEARCH_VERSION:-opensearch}.yml:ro + - ../services/elasticsearch/config/log4j2_${ELASTICSEARCH_VERSION:-opensearch}.properties:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/config-input/log4j2.properties:ro + # Supply Performance Analyzer files missing from some OpenSearch images. + - ../services/elasticsearch/config/performance-analyzer/performance-analyzer.properties:/usr/share/opensearch/config-input/opensearch-performance-analyzer/performance-analyzer.properties:ro + - ../services/elasticsearch/config/performance-analyzer/plugin-stats-metadata:/usr/share/opensearch/config-input/opensearch-performance-analyzer/plugin-stats-metadata:ro + - ../services/elasticsearch/config/performance-analyzer/agent-stats-metadata:/usr/share/opensearch/config-input/opensearch-performance-analyzer/agent-stats-metadata:ro + # Shared root CA + admin certs + - ../security/certificates/elastic/${ELASTICSEARCH_VERSION:-opensearch}/elastic-stack-ca.crt.pem:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/config-input/root-ca.crt:ro + # OPENSEARCH specific (always mounted even if unused) + - ../security/certificates/elastic/opensearch/admin.crt:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/config-input/admin.crt:ro + - ../security/certificates/elastic/opensearch/admin.key.pem:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/config-input/admin.key.pem:ro + # Shared roles + - ../security/es_roles/elasticsearch/role_mapping.yml:/usr/share/elasticsearch/config-input/role_mapping.yml:ro + - ../security/es_roles/elasticsearch/roles.yml:/usr/share/elasticsearch/config-input/roles.yml:ro + - ../security/es_roles/opensearch/config.yml:/usr/share/opensearch/config-input/opensearch-security/config.yml:ro + - ../security/es_roles/opensearch/internal_users.yml:/usr/share/opensearch/config-input/opensearch-security/internal_users.yml:ro + - ../security/es_roles/opensearch/roles_mapping.yml:/usr/share/opensearch/config-input/opensearch-security/roles_mapping.yml:ro + - ../security/es_roles/opensearch/roles.yml:/usr/share/opensearch/config-input/opensearch-security/roles.yml:ro + # Shared local ML model repository. Register files with a + # file:///usr/share/opensearch/models/.zip URL. + - ../services/elasticsearch/models:/usr/share/opensearch/models:ro + # Shared system settings + - ../services/elasticsearch/sysctl.conf:/etc/sysctl.conf:ro + # Shared snapshot mounts + - ${ELASTICSEARCH_BACKUP_PARTITION_FULL:-../data/es_snapshot_backups/full_backup}:/mnt/es_data_backups:rw + - ${ELASTICSEARCH_BACKUP_PARTITION_CONFIG:-../data/es_snapshot_backups/config_backup}:/mnt/es_config_backups:rw + + metricbeat-common: &metricbeat-common-volumes + - ../services/metricbeat/metricbeat.yml:/usr/share/metricbeat/metricbeat.yml:ro + - ../security/certificates/elastic/elasticsearch/elastic-stack-ca.crt.pem:/usr/share/metricbeat/root-ca.crt:ro + + filebeat-common: &filebeat-common-volumes + - ../services/filebeat/filebeat.yml:/usr/share/filebeat/filebeat.yml:ro + - ../security/certificates/elastic/elasticsearch/elastic-stack-ca.crt.pem:/etc/pki/root/root-ca.crt:ro + + samples-db: &samples-db-volumes + # Editable bootstrap script, schemas, and dumps share one read-only mount. + - ../services/pgsamples:/docker-entrypoint-initdb.d:ro + # data persistence + - samples-vol:/var/lib/postgresql + - type: bind + source: ${POSTGRES_SAMPLES_TLS_DIR:-../security/certificates/postgresql/samples} + target: /run/database-tls + read_only: true + bind: + create_host_path: false + + cogstack-databank-db: &cogstack-databank-db-volumes + - ../services/cogstack-db/pgsql:/docker-entrypoint-initdb.d:ro + # data persistence + - databank-vol:/var/lib/postgresql + - type: bind + source: ${POSTGRES_DATABANK_TLS_DIR:-../security/certificates/postgresql/databank} + target: /run/database-tls + read_only: true + bind: + create_host_path: false + + cogstack-databank-db-mssql: &cogstack-databank-db-mssql-volumes + # Keep SQL Server files separate from the PostgreSQL databank volume. + - databank-mssql-vol:/var/opt/mssql + + cogstack-databank-db-mssql-init: &cogstack-databank-db-mssql-init-volumes + - ../services/cogstack-db/mssql:/bootstrap:ro + + clickhouse: &clickhouse-volumes + - clickhouse-data:/var/lib/clickhouse + + opensearch-performance-analyzer-init: &opensearch-performance-analyzer-init-volumes + - ${ELASTICSEARCH_NODE_1_PA_VOL_NAME:-elasticsearch-vol-performance-analyzer-1}:/pa-1 + - ${ELASTICSEARCH_NODE_2_PA_VOL_NAME:-elasticsearch-vol-performance-analyzer-2}:/pa-2 + - ${ELASTICSEARCH_NODE_3_PA_VOL_NAME:-elasticsearch-vol-performance-analyzer-3}:/pa-3 + + es-native-create-certs: &es-native-create-certs-volumes + - ../security/scripts/es_native_cert_generator.sh:/usr/share/elasticsearch/es_native_cert_generator.sh:ro + - ../security/certificates/elastic/elasticsearch:/usr/share/elasticsearch/config/certificates:rw + - elasticsearch-certs-vol:/certs + + elasticsearch-1: &elasticsearch-1-volumes + - ../security/certificates/elastic/${ELASTICSEARCH_VERSION:-opensearch}/elasticsearch/${ES_INSTANCE_NAME_1:-elasticsearch-1}/${ES_INSTANCE_NAME_1:-elasticsearch-1}.p12:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/config-input/esnode.p12:ro + - ../security/certificates/elastic/${ELASTICSEARCH_VERSION:-opensearch}/elasticsearch/${ES_INSTANCE_NAME_1:-elasticsearch-1}/${ES_INSTANCE_NAME_1:-elasticsearch-1}.crt:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/config-input/esnode.crt:ro + - ../security/certificates/elastic/${ELASTICSEARCH_VERSION:-opensearch}/elasticsearch/${ES_INSTANCE_NAME_1:-elasticsearch-1}/${ES_INSTANCE_NAME_1:-elasticsearch-1}.key:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/config-input/esnode.key:ro + # ES data persistence + - ${ELASTICSEARCH_NODE_1_DATA_VOL_NAME:-elasticsearch-vol-1}:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/data + # ES logs + - ${ELASTICSEARCH_NODE_1_LOG_VOL_NAME:-elasticsearch-vol-log-1}:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/logs + # Performance Analyzer metrics database; metadata above is a regular config file. + - ${ELASTICSEARCH_NODE_1_PA_VOL_NAME:-elasticsearch-vol-performance-analyzer-1}:/var/lib/opensearch-performance-analyzer:rw + + elasticsearch-2: &elasticsearch-2-volumes + - ../security/certificates/elastic/${ELASTICSEARCH_VERSION:-opensearch}/elasticsearch/${ES_INSTANCE_NAME_2:-elasticsearch-2}/${ES_INSTANCE_NAME_2:-elasticsearch-2}.p12:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/config-input/esnode.p12:ro + - ../security/certificates/elastic/${ELASTICSEARCH_VERSION:-opensearch}/elasticsearch/${ES_INSTANCE_NAME_2:-elasticsearch-2}/${ES_INSTANCE_NAME_2:-elasticsearch-2}.crt:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/config-input/esnode.crt:ro + - ../security/certificates/elastic/${ELASTICSEARCH_VERSION:-opensearch}/elasticsearch/${ES_INSTANCE_NAME_2:-elasticsearch-2}/${ES_INSTANCE_NAME_2:-elasticsearch-2}.key:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/config-input/esnode.key:ro + # ES data persistence + - ${ELASTICSEARCH_NODE_2_DATA_VOL_NAME:-elasticsearch-vol-2}:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/data + # ES logs + - ${ELASTICSEARCH_NODE_2_LOG_VOL_NAME:-elasticsearch-vol-log-2}:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/logs + # Performance Analyzer metrics database; metadata above is a regular config file. + - ${ELASTICSEARCH_NODE_2_PA_VOL_NAME:-elasticsearch-vol-performance-analyzer-2}:/var/lib/opensearch-performance-analyzer:rw + + elasticsearch-3: &elasticsearch-3-volumes + - ../security/certificates/elastic/${ELASTICSEARCH_VERSION:-opensearch}/elasticsearch/${ES_INSTANCE_NAME_3:-elasticsearch-3}/${ES_INSTANCE_NAME_3:-elasticsearch-3}.p12:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/config-input/esnode.p12:ro + - ../security/certificates/elastic/${ELASTICSEARCH_VERSION:-opensearch}/elasticsearch/${ES_INSTANCE_NAME_3:-elasticsearch-3}/${ES_INSTANCE_NAME_3:-elasticsearch-3}.crt:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/config-input/esnode.crt:ro + - ../security/certificates/elastic/${ELASTICSEARCH_VERSION:-opensearch}/elasticsearch/${ES_INSTANCE_NAME_3:-elasticsearch-3}/${ES_INSTANCE_NAME_3:-elasticsearch-3}.key:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/config-input/esnode.key:ro + # ES data persistence + - ${ELASTICSEARCH_NODE_3_DATA_VOL_NAME:-elasticsearch-vol-3}:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/data + # ES logs + - ${ELASTICSEARCH_NODE_3_LOG_VOL_NAME:-elasticsearch-vol-log-3}:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/logs + # Performance Analyzer metrics database; metadata above is a regular config file. + - ${ELASTICSEARCH_NODE_3_PA_VOL_NAME:-elasticsearch-vol-performance-analyzer-3}:/var/lib/opensearch-performance-analyzer:rw + + metricbeat-1: &metricbeat-1-volumes + - metricbeat-data-1:/usr/share/metricbeat/data + - ../security/certificates/elastic/elasticsearch/elasticsearch/${ES_INSTANCE_NAME_1:-elasticsearch-1}/${ES_INSTANCE_NAME_1:-elasticsearch-1}.p12:/usr/share/metricbeat/esnode.p12:ro + - ../security/certificates/elastic/elasticsearch/elasticsearch/${ES_INSTANCE_NAME_1:-elasticsearch-1}/${ES_INSTANCE_NAME_1:-elasticsearch-1}.crt:/usr/share/metricbeat/esnode.crt:ro + - ../security/certificates/elastic/elasticsearch/elasticsearch/${ES_INSTANCE_NAME_1:-elasticsearch-1}/${ES_INSTANCE_NAME_1:-elasticsearch-1}.key:/usr/share/metricbeat/esnode.key:ro + + metricbeat-2: &metricbeat-2-volumes + - metricbeat-data-2:/usr/share/metricbeat/data + - ../security/certificates/elastic/elasticsearch/elasticsearch/${ES_INSTANCE_NAME_2:-elasticsearch-2}/${ES_INSTANCE_NAME_2:-elasticsearch-2}.p12:/usr/share/metricbeat/esnode.p12:ro + - ../security/certificates/elastic/elasticsearch/elasticsearch/${ES_INSTANCE_NAME_2:-elasticsearch-2}/${ES_INSTANCE_NAME_2:-elasticsearch-2}.crt:/usr/share/metricbeat/esnode.crt:ro + - ../security/certificates/elastic/elasticsearch/elasticsearch/${ES_INSTANCE_NAME_2:-elasticsearch-2}/${ES_INSTANCE_NAME_2:-elasticsearch-2}.key:/usr/share/metricbeat/esnode.key:ro + + metricbeat-3: &metricbeat-3-volumes + - metricbeat-data-3:/usr/share/metricbeat/data + - ../security/certificates/elastic/elasticsearch/elasticsearch/${ES_INSTANCE_NAME_3:-elasticsearch-3}/${ES_INSTANCE_NAME_3:-elasticsearch-3}.p12:/usr/share/metricbeat/esnode.p12:ro + - ../security/certificates/elastic/elasticsearch/elasticsearch/${ES_INSTANCE_NAME_3:-elasticsearch-3}/${ES_INSTANCE_NAME_3:-elasticsearch-3}.crt:/usr/share/metricbeat/esnode.crt:ro + - ../security/certificates/elastic/elasticsearch/elasticsearch/${ES_INSTANCE_NAME_3:-elasticsearch-3}/${ES_INSTANCE_NAME_3:-elasticsearch-3}.key:/usr/share/metricbeat/esnode.key:ro + + filebeat-1: &filebeat-1-volumes + - filebeat-data-1:/usr/share/filebeat/data + - ../security/certificates/elastic/elasticsearch/elasticsearch/${ES_INSTANCE_NAME_1:-elasticsearch-1}/${ES_INSTANCE_NAME_1:-elasticsearch-1}.p12:/etc/pki/client/esnode.p12:ro + - ../security/certificates/elastic/elasticsearch/elasticsearch/${ES_INSTANCE_NAME_1:-elasticsearch-1}/${ES_INSTANCE_NAME_1:-elasticsearch-1}.crt:/etc/pki/client/esnode.crt:ro + - ../security/certificates/elastic/elasticsearch/elasticsearch/${ES_INSTANCE_NAME_1:-elasticsearch-1}/${ES_INSTANCE_NAME_1:-elasticsearch-1}.key:/etc/pki/client/esnode.key:ro + - ${ELASTICSEARCH_NODE_1_LOG_VOL_NAME:-elasticsearch-vol-log-1}:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/logs + + filebeat-2: &filebeat-2-volumes + - filebeat-data-2:/usr/share/filebeat/data + - ../security/certificates/elastic/elasticsearch/elasticsearch/${ES_INSTANCE_NAME_2:-elasticsearch-2}/${ES_INSTANCE_NAME_2:-elasticsearch-2}.p12:/etc/pki/client/esnode.p12:ro + - ../security/certificates/elastic/elasticsearch/elasticsearch/${ES_INSTANCE_NAME_2:-elasticsearch-2}/${ES_INSTANCE_NAME_2:-elasticsearch-2}.crt:/etc/pki/client/esnode.crt:ro + - ../security/certificates/elastic/elasticsearch/elasticsearch/${ES_INSTANCE_NAME_2:-elasticsearch-2}/${ES_INSTANCE_NAME_2:-elasticsearch-2}.key:/etc/pki/client/esnode.key:ro + - ${ELASTICSEARCH_NODE_2_LOG_VOL_NAME:-elasticsearch-vol-log-2}:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/logs + + filebeat-3: &filebeat-3-volumes + - filebeat-data-3:/usr/share/filebeat/data + - ../security/certificates/elastic/elasticsearch/elasticsearch/${ES_INSTANCE_NAME_3:-elasticsearch-3}/${ES_INSTANCE_NAME_3:-elasticsearch-3}.p12:/etc/pki/client/esnode.p12:ro + - ../security/certificates/elastic/elasticsearch/elasticsearch/${ES_INSTANCE_NAME_3:-elasticsearch-3}/${ES_INSTANCE_NAME_3:-elasticsearch-3}.crt:/etc/pki/client/esnode.crt:ro + - ../security/certificates/elastic/elasticsearch/elasticsearch/${ES_INSTANCE_NAME_3:-elasticsearch-3}/${ES_INSTANCE_NAME_3:-elasticsearch-3}.key:/etc/pki/client/esnode.key:ro + - ${ELASTICSEARCH_NODE_3_LOG_VOL_NAME:-elasticsearch-vol-log-3}:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/logs + + kibana: &kibana-volumes + # INFO: Kibana configuration mapped via volume (make sure to comment this and uncomment the next line if you are using NATIVE kibana deployment) + - ../services/kibana/config/${ELASTICSEARCH_VERSION:-opensearch}.yml:/usr/share/${KIBANA_VERSION:-opensearch-dashboards}/config/${KIBANA_CONFIG_FILE_VERSION:-opensearch_dashboards}.yml:ro + + # Security certificates, general + - ../security/certificates/elastic/${ELASTICSEARCH_VERSION:-opensearch}/elastic-stack-ca.crt.pem:/usr/share/${KIBANA_VERSION:-opensearch-dashboards}/config/root-ca.crt:ro + + - ../security/certificates/elastic/${ELASTICSEARCH_VERSION:-opensearch}/elasticsearch/${ES_INSTANCE_NAME_1:-elasticsearch-1}/${ES_INSTANCE_NAME_1:-elasticsearch-1}.crt:/usr/share/${KIBANA_VERSION:-opensearch-dashboards}/config/esnode1.crt:ro + - ../security/certificates/elastic/${ELASTICSEARCH_VERSION:-opensearch}/elasticsearch/${ES_INSTANCE_NAME_1:-elasticsearch-1}/${ES_INSTANCE_NAME_1:-elasticsearch-1}.key:/usr/share/${KIBANA_VERSION:-opensearch-dashboards}/config/esnode1.key:ro + - ../security/certificates/elastic/${ELASTICSEARCH_VERSION:-opensearch}/elasticsearch/${ES_INSTANCE_NAME_1:-elasticsearch-1}/${ES_INSTANCE_NAME_1:-elasticsearch-1}.p12:/usr/share/${KIBANA_VERSION:-opensearch-dashboards}/config/esnode1.p12:ro + + # OpenSearch uses some default certs + - ../security/certificates/elastic/opensearch/elastic-stack-ca.crt.pem:/certificates/elastic/opensearch/elastic-stack-ca.crt.pem:ro + - ../security/certificates/elastic/opensearch/es_kibana_client.pem:/certificates/elastic/opensearch/es_kibana_client.pem:ro + - ../security/certificates/elastic/opensearch/es_kibana_client.key:/certificates/elastic/opensearch/es_kibana_client.key:ro + + ollama: &ollama-volumes + - ollama-models:/root/.ollama + + open-webui: &open-webui-volumes + - openwebui-data:/app/backend/data + + nifi-nginx-tmpfs: &nifi-nginx-tmpfs + # The published image uses nginx's stock entrypoint, which does not create this parent. + - "${NGINX_RUNTIME_DIR:-/tmp/nginx}:mode=1777" + + nifi-nginx: &nifi-nginx-volumes + - ../services/nginx/sites-enabled:/etc/nginx/sites-enabled:ro + - ../services/nginx/config/nginx.conf.template:/etc/nginx/templates/nginx.conf.template:ro + - ../security/certificates/nifi:/certificates/nifi:ro + - ../security/certificates/root/root-ca.pem:/certificates/root/root-ca.pem:ro + + gitea: &gitea-volumes + # app config + - ../services/gitea/app.ini:/etc/gitea/app.ini:rw + # dedicated leaf certificate only; never expose the root CA private key + - ../security/certificates/gitea:/certificates/gitea:ro + # data & conf volumes + - gitea-config-vol:/etc/gitea:rw + - gitea-lib-vol:/var/lib/gitea:rw + - gitea-data-vol:/data:rw + +x-networks: + cognet: &cognet + - cognet x-common-hosts: &common-hosts - ${ELASTICSEARCH_1_HOST_NAME:-test-1:0.0.0.0} @@ -74,66 +433,7 @@ x-nifi-common: &nifi-common restart: unless-stopped env_file: *all-env extra_hosts: *common-hosts - networks: - - cognet - -x-nifi-volumes: &nifi-volumes - # Drivers - - ../nifi/drivers:/opt/nifi/drivers:ro - - # User overrides bundled in the image - - ../nifi/user_scripts:/opt/nifi/user_scripts:rw - - ../nifi/user_schemas:/opt/nifi/user_schemas:rw - - # Live Python package imports - - ../nifi/user_scripts/:/opt/nifi/nifi-current/python/framework/nifi/user_scripts:rw - - # Python processors (NiFi 2.x) - - ../nifi/user_python_extensions:/opt/nifi/nifi-current/python_extensions:rw - - # Security certificates - - ../security/certificates/nifi:/security/certificates/nifi:ro - - ../security/certificates/elastic/opensearch/elasticsearch:/security/certificates/elastic/opensearch/elasticsearch:ro - - ../security/certificates/elastic/opensearch/elastic-stack-ca.crt.pem:/security/certificates/elastic/opensearch/elastic-stack-ca.crt.pem:ro - - ../security/certificates/elastic/opensearch/es_kibana_client.pem:/security/certificates/elastic/opensearch/es_kibana_client.pem:ro - - ../security/certificates/elastic/opensearch/es_kibana_client.key:/security/certificates/elastic/opensearch/es_kibana_client.key:ro - - ../security/certificates/root/root-ca.pem:/security/certificates/root/root-ca.pem:ro - # Database clients receive only the public CA from each TLS bundle. - - type: bind - source: ${POSTGRES_SAMPLES_TLS_DIR:-../security/certificates/postgresql/samples}/ca.crt - target: /run/database-cas/samples-ca.crt - read_only: true - bind: - create_host_path: false - - type: bind - source: ${POSTGRES_DATABANK_TLS_DIR:-../security/certificates/postgresql/databank}/ca.crt - target: /run/database-cas/databank-ca.crt - read_only: true - bind: - create_host_path: false - - # Security helper scripts - - ../security/scripts/nifi_create_single_user_auth.sh:/opt/nifi/nifi-current/security_scripts/nifi_create_single_user_auth.sh:ro - - # NiFi configuration - - ../nifi/conf/:/opt/nifi/nifi-current/conf/:rw - - # Ingest data directory - - ${NIFI_DATA_PATH:-../data/}:/data/:rw - - # MedCAT models - - ${NIFI_MEDCAT_SERVICE_MODEL_PRODUCTION_PATH:-../services/cogstack-nlp/medcat-service/models/}:/opt/models:rw - - # NiFi repositories/state - - nifi-vol-logs:/opt/nifi/nifi-current/logs - - nifi-vol-provenance:/opt/nifi/nifi-current/provenance_repository - - nifi-vol-database:/opt/nifi/nifi-current/database_repository - - nifi-vol-flowfiles:/opt/nifi/nifi-current/flowfile_repository - - nifi-vol-content:/opt/nifi/nifi-current/content_repository - - nifi-vol-state:/opt/nifi/nifi-current/state - - # Flowfile error output - - nifi-vol-errors:/opt/nifi/pipeline/flowfile-errors + networks: *cognet x-db-common: &db-common <<: *common-ulimits @@ -178,47 +478,15 @@ x-postgres-common: &postgres-common - -c - hba_file=/docker-entrypoint-initdb.d/access/pg_hba_tls.conf -x-es-common-volumes: &es-common-volumes - # Shared configs - - ../services/elasticsearch/search-entrypoint.sh:/usr/local/bin/cogstack-search-entrypoint.sh:ro - - ../services/elasticsearch/config/${ELASTICSEARCH_VERSION:-opensearch}.yml:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/config-input/${ELASTICSEARCH_VERSION:-opensearch}.yml:ro - - ../services/elasticsearch/config/log4j2_${ELASTICSEARCH_VERSION:-opensearch}.properties:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/config-input/log4j2.properties:ro - # Supply Performance Analyzer files missing from some OpenSearch images. - - ../services/elasticsearch/config/performance-analyzer/performance-analyzer.properties:/usr/share/opensearch/config-input/opensearch-performance-analyzer/performance-analyzer.properties:ro - - ../services/elasticsearch/config/performance-analyzer/plugin-stats-metadata:/usr/share/opensearch/config-input/opensearch-performance-analyzer/plugin-stats-metadata:ro - - ../services/elasticsearch/config/performance-analyzer/agent-stats-metadata:/usr/share/opensearch/config-input/opensearch-performance-analyzer/agent-stats-metadata:ro - # Shared root CA + admin certs - - ../security/certificates/elastic/${ELASTICSEARCH_VERSION:-opensearch}/elastic-stack-ca.crt.pem:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/config-input/root-ca.crt:ro - # OPENSEARCH specific (always mounted even if unused) - - ../security/certificates/elastic/opensearch/admin.crt:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/config-input/admin.crt:ro - - ../security/certificates/elastic/opensearch/admin.key.pem:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/config-input/admin.key.pem:ro - # Shared roles - - ../security/es_roles/elasticsearch/role_mapping.yml:/usr/share/elasticsearch/config-input/role_mapping.yml:ro - - ../security/es_roles/elasticsearch/roles.yml:/usr/share/elasticsearch/config-input/roles.yml:ro - - ../security/es_roles/opensearch/config.yml:/usr/share/opensearch/config-input/opensearch-security/config.yml:ro - - ../security/es_roles/opensearch/internal_users.yml:/usr/share/opensearch/config-input/opensearch-security/internal_users.yml:ro - - ../security/es_roles/opensearch/roles_mapping.yml:/usr/share/opensearch/config-input/opensearch-security/roles_mapping.yml:ro - - ../security/es_roles/opensearch/roles.yml:/usr/share/opensearch/config-input/opensearch-security/roles.yml:ro - # Shared local ML model repository. Register files with a - # file:///usr/share/opensearch/models/.zip URL. - - ../services/elasticsearch/models:/usr/share/opensearch/models:ro - # Shared system settings - - ../services/elasticsearch/sysctl.conf:/etc/sysctl.conf:ro - # Shared snapshot mounts - - ${ELASTICSEARCH_BACKUP_PARTITION_FULL:-../data/es_snapshot_backups/full_backup}:/mnt/es_data_backups:rw - - ${ELASTICSEARCH_BACKUP_PARTITION_CONFIG:-../data/es_snapshot_backups/config_backup}:/mnt/es_config_backups:rw - x-es-common: &es-common <<: *common-ulimits image: ${ELASTICSEARCH_DOCKER_IMAGE:?Source deploy/export_env_vars.sh first} entrypoint: ["/bin/bash", "/usr/local/bin/cogstack-search-entrypoint.sh"] - environment: - ELASTICSEARCH_VERSION: ${ELASTICSEARCH_VERSION:-opensearch} + environment: *es-common-env shm_size: ${ELASTICSEARCH_DOCKER_SHM_SIZE:-1g} restart: unless-stopped env_file: *es-env - networks: - - cognet + networks: *cognet extra_hosts: *common-hosts logging: *es-logging-common deploy: @@ -236,9 +504,7 @@ x-metricbeat-common: &metricbeat-common command: -e --strict.perms=false restart: unless-stopped shm_size: ${METRICBEAT_DOCKER_SHM:-1g} - env_file: - - "${COGSTACK_ELASTICSEARCH_ENV_FILE:-./elasticsearch.env}" - - "${COGSTACK_USERS_ELASTICSEARCH_ENV_FILE:-../security/env/users_elasticsearch.env}" + env_file: *beats-env-files deploy: resources: limits: @@ -247,11 +513,8 @@ x-metricbeat-common: &metricbeat-common reservations: cpus: "${METRICBEAT_DOCKER_CPU_MIN}" memory: "${METRICBEAT_DOCKER_RAM}" - volumes: - - ../services/metricbeat/metricbeat.yml:/usr/share/metricbeat/metricbeat.yml:ro - - ../security/certificates/elastic/elasticsearch/elastic-stack-ca.crt.pem:/usr/share/metricbeat/root-ca.crt:ro - networks: - - cognet + volumes: *metricbeat-common-volumes + networks: *cognet extra_hosts: *common-hosts logging: *es-logging-common @@ -261,9 +524,7 @@ x-filebeat-common: &filebeat-common command: ${FILEBEAT_STARTUP_COMMAND:-'-e --strict.perms=false'} restart: unless-stopped shm_size: ${FILEBEAT_DOCKER_SHM:-1g} - env_file: - - "${COGSTACK_ELASTICSEARCH_ENV_FILE:-./elasticsearch.env}" - - "${COGSTACK_USERS_ELASTICSEARCH_ENV_FILE:-../security/env/users_elasticsearch.env}" + env_file: *beats-env-files deploy: resources: limits: @@ -272,11 +533,8 @@ x-filebeat-common: &filebeat-common reservations: cpus: "${FILEBEAT_DOCKER_CPU_MIN}" memory: "${FILEBEAT_DOCKER_RAM}" - volumes: - - ../services/filebeat/filebeat.yml:/usr/share/filebeat/filebeat.yml:ro - - ../security/certificates/elastic/elasticsearch/elastic-stack-ca.crt.pem:/etc/pki/root/root-ca.crt:ro - networks: - - cognet + volumes: *filebeat-common-volumes + networks: *cognet extra_hosts: *common-hosts logging: *es-logging-common @@ -291,101 +549,45 @@ services: samples-db: <<: *postgres-common container_name: cogstack-samples-db - environment: - PGSSLMODE: verify-full - PGSSLROOTCERT: /run/postgresql/tls/ca.crt - POSTGRES_USER: ${POSTGRES_USER_SAMPLES:-test} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD_SAMPLES:?Set the sample bootstrap password} - POSTGRES_DB: ${POSTGRES_SAMPLES_DB_NAME:-db_samples} - COGSTACK_INGEST_USER: ${POSTGRES_INGEST_USER_SAMPLES:-cogstack_samples} - COGSTACK_INGEST_PASSWORD: ${POSTGRES_INGEST_PASSWORD_SAMPLES:?Set the sample ingestion password} - volumes: - # Editable bootstrap script, schemas, and dumps share one read-only mount. - - ../services/pgsamples:/docker-entrypoint-initdb.d:ro - # data persistence - - samples-vol:/var/lib/postgresql - - type: bind - source: ${POSTGRES_SAMPLES_TLS_DIR:-../security/certificates/postgresql/samples} - target: /run/database-tls - read_only: true - bind: - create_host_path: false + environment: *samples-db-env + volumes: *samples-db-volumes ports: - "${POSTGRES_SAMPLES_OUTPUT_HOST:-127.0.0.1}:${POSTGRES_SAMPLES_OUTPUT_PORT:-5554}:5432" expose: - 5432 - networks: - - cognet - + networks: *cognet #---------------------------------------------------------------------------# # CogStack Databank / Cogstack-DB, production database # #---------------------------------------------------------------------------# cogstack-databank-db: <<: *postgres-common container_name: cogstack-production-databank-db - environment: - PGSSLMODE: verify-full - PGSSLROOTCERT: /run/postgresql/tls/ca.crt - POSTGRES_USER: ${DATABASE_USER:-admin} - POSTGRES_PASSWORD: ${DATABASE_PASSWORD:?Set the PostgreSQL bootstrap password} - POSTGRES_DB: ${DATABASE_DB_NAME:-cogstack} - POSTGRES_DB_SCHEMA_PREFIX: ${POSTGRES_DB_SCHEMA_PREFIX:-cogstack_db} - COGSTACK_INGEST_USER: ${DATABASE_INGEST_USER:-cogstack_ingest} - COGSTACK_INGEST_PASSWORD: ${DATABASE_INGEST_PASSWORD:?Set the PostgreSQL ingestion password} - volumes: - - ../services/cogstack-db/pgsql:/docker-entrypoint-initdb.d:ro - # data persistence - - databank-vol:/var/lib/postgresql - - type: bind - source: ${POSTGRES_DATABANK_TLS_DIR:-../security/certificates/postgresql/databank} - target: /run/database-tls - read_only: true - bind: - create_host_path: false + environment: *cogstack-databank-db-env + volumes: *cogstack-databank-db-volumes ports: - "${POSTGRES_DATABANK_OUTPUT_HOST:-127.0.0.1}:${POSTGRES_DATABANK_OUTPUT_PORT:-5558}:5432" expose: - 5432 - networks: - - cognet - + networks: *cognet cogstack-databank-db-mssql: <<: *db-common image: &mssql-image mcr.microsoft.com/mssql/server:2019-latest@sha256:07d7dedecdbfcf6776035dc5eefb5bfd9aa1804b2f6b99f7dedb397d8a599069 container_name: cogstack-production-databank-db-mssql - environment: - ACCEPT_EULA: "Y" - MSSQL_SA_PASSWORD: ${MSSQL_SA_PASSWORD:?Set the SQL Server bootstrap password} - volumes: - # Keep SQL Server files separate from the PostgreSQL databank volume. - - databank-mssql-vol:/var/opt/mssql + environment: *cogstack-databank-db-mssql-env + volumes: *cogstack-databank-db-mssql-volumes ports: - "${MSSQL_OUTPUT_HOST:-127.0.0.1}:${MSSQL_OUTPUT_PORT:-1433}:1433" - networks: - - cognet - + networks: *cognet cogstack-databank-db-mssql-init: image: *mssql-image restart: "no" - environment: - MSSQL_SERVER: cogstack-databank-db-mssql,1433 - MSSQL_SA_USER: ${MSSQL_SA_USER:-sa} - MSSQL_SA_PASSWORD: ${MSSQL_SA_PASSWORD:?Set the SQL Server bootstrap password} - DATABASE_DB_NAME: ${DATABASE_DB_NAME:-cogstack} - POSTGRES_DB_SCHEMA_PREFIX: ${POSTGRES_DB_SCHEMA_PREFIX:-cogstack_db} - COGSTACK_INGEST_USER: ${MSSQL_INGEST_USER:-cogstack_ingest} - COGSTACK_INGEST_PASSWORD: ${MSSQL_INGEST_PASSWORD:?Set the MSSQL ingestion password} - MSSQL_TRUST_SERVER_CERTIFICATE: ${MSSQL_TRUST_SERVER_CERTIFICATE:-true} - MSSQL_TLS_HOSTNAME: ${MSSQL_TLS_HOSTNAME:-} + environment: *cogstack-databank-db-mssql-init-env depends_on: cogstack-databank-db-mssql: condition: service_started - volumes: - - ../services/cogstack-db/mssql:/bootstrap:ro + volumes: *cogstack-databank-db-mssql-init-volumes entrypoint: ["/bin/bash", "/bootstrap/init_db.sh"] - networks: - - cognet - + networks: *cognet #---------------------------------------------------------------------------# # ClickHouse analytics database # #---------------------------------------------------------------------------# @@ -394,8 +596,7 @@ services: container_name: cogstack-clickhouse hostname: clickhouse restart: unless-stopped - env_file: - - "${COGSTACK_DATABASE_ENV_FILE:-./database.env}" + env_file: *clickhouse-env-files ulimits: nofile: soft: 262144 @@ -408,8 +609,7 @@ services: reservations: cpus: "${CLICKHOUSE_DOCKER_CPU_MIN:-0.5}" memory: "${CLICKHOUSE_DOCKER_RAM_MIN:-512m}" - volumes: - - clickhouse-data:/var/lib/clickhouse + volumes: *clickhouse-volumes ports: - "${CLICKHOUSE_OUTPUT_HOST:-127.0.0.1}:${CLICKHOUSE_HTTP_PORT:-8123}:8123" - "${CLICKHOUSE_OUTPUT_HOST:-127.0.0.1}:${CLICKHOUSE_NATIVE_PORT:-9000}:9000" @@ -422,8 +622,7 @@ services: timeout: 5s retries: 10 start_period: 20s - networks: - - cognet + networks: *cognet logging: *logging-common #---------------------------------------------------------------------------# @@ -439,11 +638,7 @@ services: - chown 1000:1000 /pa-1 /pa-2 /pa-3 restart: "no" network_mode: none - volumes: - - ${ELASTICSEARCH_NODE_1_PA_VOL_NAME:-elasticsearch-vol-performance-analyzer-1}:/pa-1 - - ${ELASTICSEARCH_NODE_2_PA_VOL_NAME:-elasticsearch-vol-performance-analyzer-2}:/pa-2 - - ${ELASTICSEARCH_NODE_3_PA_VOL_NAME:-elasticsearch-vol-performance-analyzer-3}:/pa-3 - + volumes: *opensearch-performance-analyzer-init-volumes es_native_create_certs: container_name: es_create_certs image: ${ES_NATIVE_CERTS_DOCKER_IMAGE:?Source deploy/export_env_vars.sh first} @@ -455,11 +650,7 @@ services: tty: true user: "0" working_dir: /usr/share/elasticsearch - volumes: - - ../security/scripts/es_native_cert_generator.sh:/usr/share/elasticsearch/es_native_cert_generator.sh:ro - - ../security/certificates/elastic/elasticsearch:/usr/share/elasticsearch/config/certificates:rw - - elasticsearch-certs-vol:/certs - + volumes: *es-native-create-certs-volumes elasticsearch-base: <<: *es-common profiles: ["_elasticsearch_base"] @@ -474,18 +665,8 @@ services: extends: service: elasticsearch-base container_name: elasticsearch-1 - environment: - node.name: ${ELASTICSEARCH_NODE_1_NAME:-es01} - volumes: - - ../security/certificates/elastic/${ELASTICSEARCH_VERSION:-opensearch}/elasticsearch/${ES_INSTANCE_NAME_1:-elasticsearch-1}/${ES_INSTANCE_NAME_1:-elasticsearch-1}.p12:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/config-input/esnode.p12:ro - - ../security/certificates/elastic/${ELASTICSEARCH_VERSION:-opensearch}/elasticsearch/${ES_INSTANCE_NAME_1:-elasticsearch-1}/${ES_INSTANCE_NAME_1:-elasticsearch-1}.crt:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/config-input/esnode.crt:ro - - ../security/certificates/elastic/${ELASTICSEARCH_VERSION:-opensearch}/elasticsearch/${ES_INSTANCE_NAME_1:-elasticsearch-1}/${ES_INSTANCE_NAME_1:-elasticsearch-1}.key:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/config-input/esnode.key:ro - # ES data persistence - - ${ELASTICSEARCH_NODE_1_DATA_VOL_NAME:-elasticsearch-vol-1}:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/data - # ES logs - - ${ELASTICSEARCH_NODE_1_LOG_VOL_NAME:-elasticsearch-vol-log-1}:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/logs - # Performance Analyzer metrics database; metadata above is a regular config file. - - ${ELASTICSEARCH_NODE_1_PA_VOL_NAME:-elasticsearch-vol-performance-analyzer-1}:/var/lib/opensearch-performance-analyzer:rw + environment: *elasticsearch-1-env + volumes: *elasticsearch-1-volumes ports: - "${ELASTICSEARCH_NODE_1_OUTPUT_PORT:-9200}:9200" - "${ELASTICSEARCH_NODE_1_COMM_OUTPUT_PORT:-9300}:9300" @@ -495,18 +676,8 @@ services: extends: service: elasticsearch-base container_name: elasticsearch-2 - environment: - node.name: ${ELASTICSEARCH_NODE_2_NAME:-es02} - volumes: - - ../security/certificates/elastic/${ELASTICSEARCH_VERSION:-opensearch}/elasticsearch/${ES_INSTANCE_NAME_2:-elasticsearch-2}/${ES_INSTANCE_NAME_2:-elasticsearch-2}.p12:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/config-input/esnode.p12:ro - - ../security/certificates/elastic/${ELASTICSEARCH_VERSION:-opensearch}/elasticsearch/${ES_INSTANCE_NAME_2:-elasticsearch-2}/${ES_INSTANCE_NAME_2:-elasticsearch-2}.crt:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/config-input/esnode.crt:ro - - ../security/certificates/elastic/${ELASTICSEARCH_VERSION:-opensearch}/elasticsearch/${ES_INSTANCE_NAME_2:-elasticsearch-2}/${ES_INSTANCE_NAME_2:-elasticsearch-2}.key:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/config-input/esnode.key:ro - # ES data persistence - - ${ELASTICSEARCH_NODE_2_DATA_VOL_NAME:-elasticsearch-vol-2}:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/data - # ES logs - - ${ELASTICSEARCH_NODE_2_LOG_VOL_NAME:-elasticsearch-vol-log-2}:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/logs - # Performance Analyzer metrics database; metadata above is a regular config file. - - ${ELASTICSEARCH_NODE_2_PA_VOL_NAME:-elasticsearch-vol-performance-analyzer-2}:/var/lib/opensearch-performance-analyzer:rw + environment: *elasticsearch-2-env + volumes: *elasticsearch-2-volumes ports: - "${ELASTICSEARCH_NODE_2_OUTPUT_PORT:-9201}:9200" - "${ELASTICSEARCH_NODE_2_COMM_OUTPUT_PORT:-9301}:9300" @@ -516,18 +687,8 @@ services: extends: service: elasticsearch-base container_name: elasticsearch-3 - environment: - node.name: ${ELASTICSEARCH_NODE_3_NAME:-es03} - volumes: - - ../security/certificates/elastic/${ELASTICSEARCH_VERSION:-opensearch}/elasticsearch/${ES_INSTANCE_NAME_3:-elasticsearch-3}/${ES_INSTANCE_NAME_3:-elasticsearch-3}.p12:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/config-input/esnode.p12:ro - - ../security/certificates/elastic/${ELASTICSEARCH_VERSION:-opensearch}/elasticsearch/${ES_INSTANCE_NAME_3:-elasticsearch-3}/${ES_INSTANCE_NAME_3:-elasticsearch-3}.crt:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/config-input/esnode.crt:ro - - ../security/certificates/elastic/${ELASTICSEARCH_VERSION:-opensearch}/elasticsearch/${ES_INSTANCE_NAME_3:-elasticsearch-3}/${ES_INSTANCE_NAME_3:-elasticsearch-3}.key:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/config-input/esnode.key:ro - # ES data persistence - - ${ELASTICSEARCH_NODE_3_DATA_VOL_NAME:-elasticsearch-vol-3}:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/data - # ES logs - - ${ELASTICSEARCH_NODE_3_LOG_VOL_NAME:-elasticsearch-vol-log-3}:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/logs - # Performance Analyzer metrics database; metadata above is a regular config file. - - ${ELASTICSEARCH_NODE_3_PA_VOL_NAME:-elasticsearch-vol-performance-analyzer-3}:/var/lib/opensearch-performance-analyzer:rw + environment: *elasticsearch-3-env + volumes: *elasticsearch-3-volumes ports: - "${ELASTICSEARCH_NODE_3_OUTPUT_PORT:-9202}:9200" - "${ELASTICSEARCH_NODE_3_COMM_OUTPUT_PORT:-9302}:9300" @@ -536,60 +697,27 @@ services: metricbeat-1: <<: *metricbeat-common container_name: cogstack-metricbeat-1 - volumes: - - metricbeat-data-1:/usr/share/metricbeat/data - - ../security/certificates/elastic/elasticsearch/elasticsearch/${ES_INSTANCE_NAME_1:-elasticsearch-1}/${ES_INSTANCE_NAME_1:-elasticsearch-1}.p12:/usr/share/metricbeat/esnode.p12:ro - - ../security/certificates/elastic/elasticsearch/elasticsearch/${ES_INSTANCE_NAME_1:-elasticsearch-1}/${ES_INSTANCE_NAME_1:-elasticsearch-1}.crt:/usr/share/metricbeat/esnode.crt:ro - - ../security/certificates/elastic/elasticsearch/elasticsearch/${ES_INSTANCE_NAME_1:-elasticsearch-1}/${ES_INSTANCE_NAME_1:-elasticsearch-1}.key:/usr/share/metricbeat/esnode.key:ro - + volumes: *metricbeat-1-volumes metricbeat-2: <<: *metricbeat-common container_name: cogstack-metricbeat-2 - volumes: - - metricbeat-data-2:/usr/share/metricbeat/data - - ../security/certificates/elastic/elasticsearch/elasticsearch/${ES_INSTANCE_NAME_2:-elasticsearch-2}/${ES_INSTANCE_NAME_2:-elasticsearch-2}.p12:/usr/share/metricbeat/esnode.p12:ro - - ../security/certificates/elastic/elasticsearch/elasticsearch/${ES_INSTANCE_NAME_2:-elasticsearch-2}/${ES_INSTANCE_NAME_2:-elasticsearch-2}.crt:/usr/share/metricbeat/esnode.crt:ro - - ../security/certificates/elastic/elasticsearch/elasticsearch/${ES_INSTANCE_NAME_2:-elasticsearch-2}/${ES_INSTANCE_NAME_2:-elasticsearch-2}.key:/usr/share/metricbeat/esnode.key:ro - + volumes: *metricbeat-2-volumes metricbeat-3: <<: *metricbeat-common container_name: cogstack-metricbeat-3 - volumes: - - metricbeat-data-3:/usr/share/metricbeat/data - - ../security/certificates/elastic/elasticsearch/elasticsearch/${ES_INSTANCE_NAME_3:-elasticsearch-3}/${ES_INSTANCE_NAME_3:-elasticsearch-3}.p12:/usr/share/metricbeat/esnode.p12:ro - - ../security/certificates/elastic/elasticsearch/elasticsearch/${ES_INSTANCE_NAME_3:-elasticsearch-3}/${ES_INSTANCE_NAME_3:-elasticsearch-3}.crt:/usr/share/metricbeat/esnode.crt:ro - - ../security/certificates/elastic/elasticsearch/elasticsearch/${ES_INSTANCE_NAME_3:-elasticsearch-3}/${ES_INSTANCE_NAME_3:-elasticsearch-3}.key:/usr/share/metricbeat/esnode.key:ro - + volumes: *metricbeat-3-volumes filebeat-1: <<: *filebeat-common container_name: cogstack-filebeat-1 - volumes: - - filebeat-data-1:/usr/share/filebeat/data - - ../security/certificates/elastic/elasticsearch/elasticsearch/${ES_INSTANCE_NAME_1:-elasticsearch-1}/${ES_INSTANCE_NAME_1:-elasticsearch-1}.p12:/etc/pki/client/esnode.p12:ro - - ../security/certificates/elastic/elasticsearch/elasticsearch/${ES_INSTANCE_NAME_1:-elasticsearch-1}/${ES_INSTANCE_NAME_1:-elasticsearch-1}.crt:/etc/pki/client/esnode.crt:ro - - ../security/certificates/elastic/elasticsearch/elasticsearch/${ES_INSTANCE_NAME_1:-elasticsearch-1}/${ES_INSTANCE_NAME_1:-elasticsearch-1}.key:/etc/pki/client/esnode.key:ro - - ${ELASTICSEARCH_NODE_1_LOG_VOL_NAME:-elasticsearch-vol-log-1}:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/logs - + volumes: *filebeat-1-volumes filebeat-2: <<: *filebeat-common container_name: cogstack-filebeat-2 - volumes: - - filebeat-data-2:/usr/share/filebeat/data - - ../security/certificates/elastic/elasticsearch/elasticsearch/${ES_INSTANCE_NAME_2:-elasticsearch-2}/${ES_INSTANCE_NAME_2:-elasticsearch-2}.p12:/etc/pki/client/esnode.p12:ro - - ../security/certificates/elastic/elasticsearch/elasticsearch/${ES_INSTANCE_NAME_2:-elasticsearch-2}/${ES_INSTANCE_NAME_2:-elasticsearch-2}.crt:/etc/pki/client/esnode.crt:ro - - ../security/certificates/elastic/elasticsearch/elasticsearch/${ES_INSTANCE_NAME_2:-elasticsearch-2}/${ES_INSTANCE_NAME_2:-elasticsearch-2}.key:/etc/pki/client/esnode.key:ro - - ${ELASTICSEARCH_NODE_2_LOG_VOL_NAME:-elasticsearch-vol-log-2}:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/logs - + volumes: *filebeat-2-volumes filebeat-3: <<: *filebeat-common container_name: cogstack-filebeat-3 - volumes: - - filebeat-data-3:/usr/share/filebeat/data - - ../security/certificates/elastic/elasticsearch/elasticsearch/${ES_INSTANCE_NAME_3:-elasticsearch-3}/${ES_INSTANCE_NAME_3:-elasticsearch-3}.p12:/etc/pki/client/esnode.p12:ro - - ../security/certificates/elastic/elasticsearch/elasticsearch/${ES_INSTANCE_NAME_3:-elasticsearch-3}/${ES_INSTANCE_NAME_3:-elasticsearch-3}.crt:/etc/pki/client/esnode.crt:ro - - ../security/certificates/elastic/elasticsearch/elasticsearch/${ES_INSTANCE_NAME_3:-elasticsearch-3}/${ES_INSTANCE_NAME_3:-elasticsearch-3}.key:/etc/pki/client/esnode.key:ro - - ${ELASTICSEARCH_NODE_3_LOG_VOL_NAME:-elasticsearch-vol-log-3}:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/logs - + volumes: *filebeat-3-volumes #---------------------------------------------------------------------------# # Kibana webapp # #---------------------------------------------------------------------------# @@ -608,25 +736,10 @@ services: reservations: cpus: "${KIBANA_DOCKER_CPU_MIN}" memory: "${KIBANA_DOCKER_RAM}" - volumes: - # INFO: Kibana configuration mapped via volume (make sure to comment this and uncomment the next line if you are using NATIVE kibana deployment) - - ../services/kibana/config/${ELASTICSEARCH_VERSION:-opensearch}.yml:/usr/share/${KIBANA_VERSION:-opensearch-dashboards}/config/${KIBANA_CONFIG_FILE_VERSION:-opensearch_dashboards}.yml:ro - - # Security certificates, general - - ../security/certificates/elastic/${ELASTICSEARCH_VERSION:-opensearch}/elastic-stack-ca.crt.pem:/usr/share/${KIBANA_VERSION:-opensearch-dashboards}/config/root-ca.crt:ro - - - ../security/certificates/elastic/${ELASTICSEARCH_VERSION:-opensearch}/elasticsearch/${ES_INSTANCE_NAME_1:-elasticsearch-1}/${ES_INSTANCE_NAME_1:-elasticsearch-1}.crt:/usr/share/${KIBANA_VERSION:-opensearch-dashboards}/config/esnode1.crt:ro - - ../security/certificates/elastic/${ELASTICSEARCH_VERSION:-opensearch}/elasticsearch/${ES_INSTANCE_NAME_1:-elasticsearch-1}/${ES_INSTANCE_NAME_1:-elasticsearch-1}.key:/usr/share/${KIBANA_VERSION:-opensearch-dashboards}/config/esnode1.key:ro - - ../security/certificates/elastic/${ELASTICSEARCH_VERSION:-opensearch}/elasticsearch/${ES_INSTANCE_NAME_1:-elasticsearch-1}/${ES_INSTANCE_NAME_1:-elasticsearch-1}.p12:/usr/share/${KIBANA_VERSION:-opensearch-dashboards}/config/esnode1.p12:ro - - # OpenSearch uses some default certs - - ../security/certificates/elastic/opensearch/elastic-stack-ca.crt.pem:/certificates/elastic/opensearch/elastic-stack-ca.crt.pem:ro - - ../security/certificates/elastic/opensearch/es_kibana_client.pem:/certificates/elastic/opensearch/es_kibana_client.pem:ro - - ../security/certificates/elastic/opensearch/es_kibana_client.key:/certificates/elastic/opensearch/es_kibana_client.key:ro + volumes: *kibana-volumes ports: - "${KIBANA_SERVER_OUTPUT_PORT:-5601}:5601" - networks: - - cognet + networks: *cognet extra_hosts: *common-hosts logging: *es-logging-common @@ -638,8 +751,7 @@ services: container_name: ${OLLAMA_CONTAINER_NAME:-cogstack-ollama} platform: ${DOCKER_DEFAULT_PLATFORM:-linux/amd64} restart: unless-stopped - env_file: - - "${COGSTACK_OLLAMA_ENV_FILE:-./ollama.env}" + env_file: *ollama-env-files deploy: resources: limits: @@ -648,15 +760,13 @@ services: reservations: cpus: "${OLLAMA_DOCKER_CPU_MIN:-1}" memory: "${OLLAMA_DOCKER_RAM_MIN:-2g}" - volumes: - - ollama-models:/root/.ollama + volumes: *ollama-volumes ports: # Keep the unauthenticated Ollama API local to the Docker host. - "${OLLAMA_OUTPUT_HOST:-127.0.0.1}:${OLLAMA_OUTPUT_PORT:-11434}:11434" expose: - "11434" - networks: - - cognet + networks: *cognet logging: *logging-common healthcheck: test: ["CMD", "ollama", "list"] @@ -680,17 +790,13 @@ services: - ALL security_opt: - no-new-privileges:true - environment: - OLLAMA_UPSTREAM_HOST: ollama - OLLAMA_UPSTREAM_PORT: 11434 - OLLAMA_UPSTREAM_TIMEOUT_SECONDS: ${OPENSEARCH_AI_READ_TIMEOUT_SECONDS:-360} + environment: *ollama-openai-proxy-env depends_on: ollama: condition: service_healthy expose: - "11435" - networks: - - cognet + networks: *cognet logging: *logging-common healthcheck: test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:11435/healthz', timeout=5)"] @@ -707,8 +813,7 @@ services: container_name: ${OPENWEBUI_CONTAINER_NAME:-cogstack-open-webui} platform: ${DOCKER_DEFAULT_PLATFORM:-linux/amd64} restart: unless-stopped - env_file: - - "${COGSTACK_OPENWEBUI_ENV_FILE:-./openwebui.env}" + env_file: *open-webui-env-files depends_on: ollama: condition: service_healthy @@ -720,15 +825,13 @@ services: reservations: cpus: "${OPENWEBUI_DOCKER_CPU_MIN:-0.25}" memory: "${OPENWEBUI_DOCKER_RAM_MIN:-256m}" - volumes: - - openwebui-data:/app/backend/data + volumes: *open-webui-volumes ports: # Port 3000 is already used by Gitea in this Compose project. - "${OPENWEBUI_OUTPUT_HOST:-127.0.0.1}:${OPENWEBUI_OUTPUT_PORT:-3001}:8080" expose: - "8080" - networks: - - cognet + networks: *cognet logging: *logging-common #---------------------------------------------------------------------------# @@ -740,10 +843,7 @@ services: container_name: cogstack-nifi hostname: nifi shm_size: ${NIFI_DOCKER_SHM_SIZE:-"1g"} - environment: - <<: *nifi-db-environment - PYTHONPATH: ${NIFI_PYTHONPATH:-/opt/nifi/nifi-current/python/framework} - JVM_OPTS: ${NIFI_JVM_OPTS:--XX:+UseG1GC -XX:MaxGCPauseMillis=200 -XX:+ParallelRefProcEnabled -Djava.security.egd=file:/dev/./urandom} + environment: *nifi-env deploy: resources: limits: @@ -754,11 +854,7 @@ services: memory: "${NIFI_DOCKER_RAM}" volumes: *nifi-volumes - # INFO : Uncomment the below line to generate your own USERNAME and PASSWORD, - # a bit messy this way as you will need to copy the credentials back - # to the "login-identity-providers.xml" section. - # entrypoint: bash -c "/opt/nifi/nifi-current/bin/nifi.sh set-single-user-credentials admin admincogstacknifi" - tty: true + stop_grace_period: 120s ports: - "${NIFI_OUTPUT_PORT:-8082}:${NIFI_INTERNAL_PORT:-8443}" - "${NIFI_INPUT_SOCKET_PORT:-10000}" @@ -769,7 +865,10 @@ services: container_name: cogstack-nifi-nginx restart: always shm_size: ${NGINX_SHM_SIZE:-1g} - env_file: *all-env + env_file: *nifi-nginx-env-files + environment: *nifi-nginx-env + tmpfs: *nifi-nginx-tmpfs + stop_grace_period: 90s deploy: resources: limits: @@ -778,15 +877,10 @@ services: reservations: cpus: "${NGINX_DOCKER_CPU_MIN}" memory: "${NGINX_DOCKER_RAM}" - volumes: - - ../services/nginx/sites-enabled:/etc/nginx/sites-enabled:ro - - ../services/nginx/config/nginx.conf.template:/etc/nginx/templates/nginx.conf.template:ro - - ../security/certificates/nifi:/certificates/nifi:ro - - ../security/certificates/root/root-ca.pem:/certificates/root/root-ca.pem:ro + volumes: *nifi-nginx-volumes ports: - "${NIFI_EXTERNAL_PORT_NGINX:-8443}:${NIFI_INTERNAL_PORT_NGINX:-8443}" - networks: - - cognet + networks: *cognet extra_hosts: *common-hosts logging: *nifi-logging-common @@ -807,17 +901,8 @@ services: reservations: cpus: "${GITEA_DOCKER_CPU_MIN}" memory: "${GITEA_DOCKER_RAM}" - volumes: - # app config - - ../services/gitea/app.ini:/etc/gitea/app.ini:rw - # dedicated leaf certificate only; never expose the root CA private key - - ../security/certificates/gitea:/certificates/gitea:ro - # data & conf volumes - - gitea-config-vol:/etc/gitea:rw - - gitea-lib-vol:/var/lib/gitea:rw - - gitea-data-vol:/data:rw - networks: - - cognet + volumes: *gitea-volumes + networks: *cognet ports: - "3000:3000" - "2222:22" diff --git a/docs/deploy/deployment.md b/docs/deploy/deployment.md index 05a3305fb..2a29aaa38 100644 --- a/docs/deploy/deployment.md +++ b/docs/deploy/deployment.md @@ -276,6 +276,10 @@ make -C deploy remote-delete-service \ See [Repeatable deployments](./reproducible.md) for image pins, submodule upgrades, and local build commands. +For rebuilt NiFi/nginx images, see [NiFi container runtime](./nifi-runtime.md) for +automatic credentials, non-root file permissions, persistent configuration and +health checks. Existing published image digests are updated as a separate release. + --- ### 🚀 Start Services diff --git a/docs/deploy/nifi-runtime.md b/docs/deploy/nifi-runtime.md new file mode 100644 index 000000000..20690d0f5 --- /dev/null +++ b/docs/deploy/nifi-runtime.md @@ -0,0 +1,174 @@ +# NiFi and nginx container runtime + +The rebuilt NiFi and `nifi-nginx` images run as non-root users and include their +baseline application resources. Their startup does not require a repository +checkout. On-prem processor/source mounts remain supported. + +These changes apply to newly built images. The published digests in +`deploy/nifi.env` remain unchanged until new images are published. Use +`make -C deploy start-nifi-dev-build` to build the current source. +Promote those images and update the published digest settings when releasing. + +## Configuration and secrets + +NiFi inherits Apache's entrypoint, Python interpreter and Python directory +defaults. The image enables certificate reload and sets shutdown periods during +the build. Deployment configuration and credentials from the checkout are not +copied into the image. + +Apache's bundled NARs are inherited; only additional extensions are downloaded. +The image retains `/tmp/cogstack_nifi/nifi` bootstrap resources consumed by the +existing chart, and the script-import link used by editable site mounts. + +The development Compose service keeps writable runtime configuration at +`/opt/nifi/nifi-current/conf` in the existing `nifi-vol-conf` volume. Docker +populates a new volume from the image; subsequent starts retain the flow, +authorization files and configuration already in that volume. The repository's +`nifi/conf` directory is not automatically imported into this volume. + +The published-image Compose service retains its writable `nifi/conf` bind mount. +A deployment can also supply its own writable configuration directory. Apache's +startup edits configuration in place, so its runtime `conf` directory must be +writable by the runtime user. In Kubernetes, any preparation of read-only +ConfigMap/Secret configuration belongs in the deployment, for example an init +container copying settings to writable storage. + +Compose maps the existing `NIFI_SINGLE_USER_CREDENTIALS_*`, `NIFI_KEYSTORE_*` +and `NIFI_TRUSTSTORE_*` settings to Apache's native environment variables. +Existing deployment security env files continue to work. Direct image users +and Kubernetes deployments should supply the native names: + +```text +AUTH=tls +NIFI_SECURITY_USER_AUTHORIZER=single-user-authorizer +NIFI_SECURITY_USER_LOGIN_IDENTITY_PROVIDER=single-user-provider +SINGLE_USER_CREDENTIALS_USERNAME= +SINGLE_USER_CREDENTIALS_PASSWORD= +KEYSTORE_PATH=/run/certificates/keystore.p12 +KEYSTORE_TYPE=PKCS12 +KEYSTORE_PASSWORD= +TRUSTSTORE_PATH=/run/certificates/truststore.p12 +TRUSTSTORE_TYPE=PKCS12 +TRUSTSTORE_PASSWORD= +NIFI_SENSITIVE_PROPS_KEY= +``` + +Here `AUTH=tls` invokes Apache's supplied-certificate configuration. The explicit +single-user authorizer and login provider preserve username/password login over +HTTPS. Other authentication modes use Apache's native settings for those modes. + +Kubernetes can populate credential environment variables with +`env.valueFrom.secretKeyRef` and mount certificate files read-only. There is no +custom `_FILE` secret loader or configuration-copying entrypoint. Use +hex-generated values for store passwords and the flow key so they pass unchanged +through Apache's property substitution. + +Keep `NIFI_SENSITIVE_PROPS_KEY` stable with the persisted flow and retain a +protected backup. Rotating it requires NiFi's encryption migration procedure. +Settings without a supported Apache environment variable are configured in the +runtime `nifi.properties` or `bootstrap.conf`. + +## Persistent and editable paths + +| Path | Purpose | +|---|---| +| `/opt/nifi/nifi-current/conf` | Runtime configuration, flow definition and authorization files | +| `/opt/nifi/nifi-current/flowfile_repository` | Queue metadata | +| `/opt/nifi/nifi-current/content_repository` | Queued content | +| `/opt/nifi/nifi-current/database_repository` | NiFi databases | +| `/opt/nifi/nifi-current/provenance_repository` | Provenance history | +| `/opt/nifi/nifi-current/state` | Component state and default script SQLite databases | +| `/opt/nifi/user_scripts` | Editable site scripts; imports follow this directory | +| `/opt/nifi/nifi-current/python_extensions` | Editable Python processor extensions | +| `/opt/nifi/nifi-current/work/python` | Generated Python environments/cache, separate from source | + +Preserve the repository volumes together with the flow definition. Replacing an +image must not replace these volumes. Each NiFi instance needs its own repository +and local-state storage. Keep logs available through container stdout and retain +log files separately when required. + +The existing script and processor mounts remain editable on the host. Mounting +them does not require rebuilding the image. Startup never copies bundled source +over a mounted site directory. Python imports of `nifi.user_scripts` follow the +mounted scripts directory, without requiring a second copy of the source. + +Published images default to UID/GID `1000:1000`. Local Compose builds use the host +UID/GID, selected by `COGSTACK_LOCAL_UID` and `COGSTACK_LOCAL_GID`, so private +host-owned certificate files remain readable without making them public. Existing +volumes must be writable by the selected runtime identity; migrate ownership of +existing NiFi volumes before switching identities. Secret/certificate inputs must +be readable by that identity or its filesystem group. Startup does not run a root +ownership-fixing process. + +Before upgrading an existing site that uses script SQLite databases, stop NiFi +and copy the contents of `nifi/user_scripts/db/` into the persistent +`state/user_scripts/db/` directory, preserving ownership for the runtime user. +Keep the original files as a backup. The new default does not automatically move +or delete site data; a site-specific `NIFI_USER_SCRIPT_DB_DIR` still overrides it. + +## nginx addressing and TLS + +nginx renders its bundled template into `/tmp/nginx/nginx.conf`. Only its runtime +directory needs writes; it supports a read-only root filesystem with writable +`/tmp`. Existing template and site mounts remain optional overrides. + +The published-image Compose service provides a temporary filesystem at +`NGINX_RUNTIME_DIR` (default `/tmp/nginx`). This also creates the parent directory +required by the older image's stock entrypoint when loading the updated template. + +| Setting | Default / purpose | +|---|---| +| `NIFI_UPSTREAM_HOST` | `nifi`; configurable DNS name | +| `NIFI_UPSTREAM_PORT` | `NIFI_INTERNAL_PORT`, otherwise `8443` | +| `NIFI_UPSTREAM_TLS_NAME` | Certificate hostname to verify, normally `nifi` | +| `NGINX_NIFI_SERVER_NAME` | Public nginx server name, default `_` | +| `NIFI_WEB_PROXY_PORT` | Public port advertised to NiFi, default `8443` | +| `NGINX_PROXY_SCHEME` | Public scheme, default `https` | +| `NGINX_PROXY_CONTEXT_PATH` | Optional public prefix before `/nifi`, empty by default | +| `NGINX_TLS_CERTIFICATE` | `/certificates/nifi/nifi.pem` | +| `NGINX_TLS_CERTIFICATE_KEY` | `/certificates/nifi/nifi.key` | +| `NGINX_UPSTREAM_CA` | `/certificates/root/root-ca.pem` | +| `NGINX_DNS_RESOLVER` | First nameserver from the container's `/etc/resolv.conf` | + +nginx verifies both NiFi's certificate chain and hostname. Its upstream DNS +address refreshes when endpoints change. Set NiFi's `NIFI_WEB_PROXY_HOST` to the +public hostname/port used by clients. For a public prefix such as `/cogstack`, set +`NGINX_PROXY_CONTEXT_PATH=/cogstack` and include `/cogstack` in NiFi's +`NIFI_WEB_PROXY_CONTEXT_PATH`; clients then use `/cogstack/nifi/`. + +There is no automatic localhost-to-loopback redirect. Open `127.0.0.1` directly +for browsers affected by localhost cookie handling. + +Mount certificate directories read-only so replacement files become visible. +New image configuration enables keystore/truststore reload through +`nifi.security.autoreload.enabled=true` in `nifi.properties`. Existing persisted +configuration retains its current value. After renewing nginx's certificate files, run +`nginx -s reload -c /tmp/nginx/nginx.conf` inside the container, or replace the +container. If `NGINX_RUNTIME_DIR` is overridden, use that directory instead. + +## Health and shutdown + +The images provide exec-compatible checks without exposing application secrets: + +```text +/opt/nifi/scripts/cogstack-healthcheck.sh live +/opt/nifi/scripts/cogstack-healthcheck.sh startup +/opt/nifi/scripts/cogstack-healthcheck.sh ready +/usr/local/bin/cogstack-nginx-healthcheck.sh live +/usr/local/bin/cogstack-nginx-healthcheck.sh ready +``` + +NiFi liveness checks its process status. Startup/readiness checks its local HTTPS +`/nifi-api/authentication/configuration` API using certificate verification. Override +`NIFI_HEALTHCHECK_HOST` and `NIFI_HEALTHCHECK_CA_FILE` when using different +certificate names/issuers. nginx liveness returns locally on +`127.0.0.1:8080/healthz/live`; readiness checks NiFi through the verified upstream. +An upstream outage does not fail nginx liveness. `NGINX_HEALTH_PORT` changes the +local check port. Docker health checks use readiness; other runtimes can call the +scripts for their separate startup/liveness/readiness policies. + +Apache's startup script handles shutdown signals. New image configuration sets +`nifi.flowcontroller.graceful.shutdown.period=30 sec` in `nifi.properties` and +`graceful.shutdown.seconds=60` in `bootstrap.conf`. Existing persisted or mounted +configuration retains its values. Compose allows 120 seconds before killing +NiFi. nginx uses graceful `SIGQUIT` shutdown and a 90-second Compose stop grace. diff --git a/docs/security/nifi.md b/docs/security/nifi.md index a6b84fc9a..a4d3c7b75 100644 --- a/docs/security/nifi.md +++ b/docs/security/nifi.md @@ -1,6 +1,7 @@ ## 🔐 NiFi TLS & Admin Access Setup (with NGINX) -This guide documents how to configure TLS and certificate-based admin access for Apache NiFi behind NGINX. +This guide covers HTTPS and single-user access to Apache NiFi behind nginx. +Client-certificate administrator identities apply when that authentication mode is selected. For background on how certificates are generated, see [Certificates and Root CA](certificates.md). --- @@ -36,7 +37,8 @@ For securing Apache NiFi endpoints with certificates, see [the official document Before starting the NiFi container: - run `make -C deploy init-security-nifi` from the repository root. This creates the shared root CA when needed, then prepares the NiFi bundle, mounted OpenSearch leaf certificates and public CA, and PostgreSQL CA files used by maintained flows. Complete existing certificate sets are preserved, including when a different search backend is selected. For supplied PostgreSQL bundles, see [Docker database security](./databases.md). -- set `nifi.sensitive.props.key` to a stable value (minimum 12 characters). +- supply a stable `NIFI_SENSITIVE_PROPS_KEY` + value (minimum 12 characters), and retain it with the persisted flow. These certificate files are required even when the database and search services run elsewhere or are not started. The combined `init-security` target covers all @@ -66,10 +68,16 @@ Local single-user credentials are defined in `security/env/users_nifi.env`. They /opt/nifi/nifi-current/bin/nifi.sh set-single-user-credentials USERNAME PASSWORD ``` -- alternatively: - - set credentials in `security/env/users_nifi.env` - - stop NiFi (`docker stop cogstack-nifi`) - - run `bash security/scripts/nifi_init_create_user_auth.sh` +- Compose passes `security/env/users_nifi.env` credentials to Apache's native + startup variables, which set the login automatically. Kubernetes deployments + can populate `SINGLE_USER_CREDENTIALS_USERNAME` and + `SINGLE_USER_CREDENTIALS_PASSWORD` through Secret environment references. +- For older published images, `bash security/scripts/nifi_init_create_user_auth.sh` + updates the running NiFi credentials and restarts NiFi. It does not delete + containers. + +See [NiFi container runtime](../deploy/nifi-runtime.md) for non-root permissions, +persistent configuration, Secret inputs, proxy TLS and health checks. URL: @@ -109,54 +117,18 @@ nifi.security.identity.mapping.pattern.dn=^.*?CN=(.*?)(,|$) ### 🌍 NGINX reverse proxy example (NiFi) -```nginx -server { - listen 8443 ssl; - server_name nginx.local; - - ssl_certificate /certificates/nifi/nifi.pem; - ssl_certificate_key /certificates/nifi/nifi.key; - ssl_client_certificate /certificates/root/root-ca.pem; - ssl_trusted_certificate /certificates/root/root-ca.pem; - - location / { - proxy_set_header Host nifi; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-ProxyHost $host; - proxy_set_header X-ProxyPort 8443; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-ProxyScheme $scheme; - proxy_pass https://nifi; - } - - location /nifi { - proxy_set_header Host nifi; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-ProxyHost $host; - proxy_set_header X-ProxyPort 8443; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-ProxyScheme $scheme; - proxy_pass https://nifi; - } - - location ^~ /nifi-api/ { - proxy_set_header Host nifi; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-ProxyHost $host; - proxy_set_header X-ProxyPort 8443; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-ProxyScheme $scheme; - proxy_pass https://nifi/nifi-api/; - } -} -``` +The maintained template is `services/nginx/config/nginx.conf.template` and is +included in rebuilt proxy images. It enables `proxy_ssl_verify on`, verifies the +configured `NIFI_UPSTREAM_TLS_NAME`, and trusts `NGINX_UPSTREAM_CA`. Public +addressing and optional context prefixes are configured through deployment +settings; see [nginx addressing and TLS](../deploy/nifi-runtime.md#nginx-addressing-and-tls). --- ### ✅ Final checklist -- [x] Certificate CN matches the NiFi admin identity -- [x] NGINX uses the expected certificates and root CA +- [x] Certificate SANs cover the public and upstream TLS hostnames +- [x] nginx verifies NiFi's certificate against the expected CA - [x] `nifi.web.proxy.*` settings match exposed URL and context - [x] Clear cookies if browser UI shows `Anonymous` @@ -166,11 +138,13 @@ server { ```bash curl --cacert security/certificates/root/root-ca.pem \ - --cert security/certificates/nifi/nifi.pem \ - --key security/certificates/nifi/nifi.key \ - https://localhost:8443/nifi-api/flow/about + https://127.0.0.1:8443/nifi-api/authentication/configuration ``` +This checks HTTPS readiness. Use username/password login for the default +single-user authentication mode. The maintained smoke test also requests a token +and checks an authenticated API endpoint. + --- ### 🛠 Troubleshooting diff --git a/mkdocs.yml b/mkdocs.yml index 8b765eaea..74e59e77e 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -47,6 +47,7 @@ nav: - 📌 Repeatable deployments: deploy/reproducible.md - ⚙️ Configuration: deploy/configuration.md - 🧩 Services: deploy/services.md + - 💧 NiFi container runtime: deploy/nifi-runtime.md - 🤖 OpenSearch AI with SGLang: deploy/sglang.md - 📛 Troubleshooting: deploy/troubleshooting.md - 🔀 Workflows: diff --git a/nifi/Dockerfile b/nifi/Dockerfile index c95cc7360..cc7e367c1 100644 --- a/nifi/Dockerfile +++ b/nifi/Dockerfile @@ -1,3 +1,4 @@ +# syntax=docker/dockerfile:1 ARG NIFI_BASE_IMAGE=apache/nifi:2.12.0@sha256:6f7f2840dbf178c6dd3fa13435b9b358f3be3c6fb7417de467a35a267f4d28ad FROM ${NIFI_BASE_IMAGE} @@ -12,34 +13,18 @@ ARG HTTPS_PROXY="" ARG no_proxy="" ARG DEBIAN_FRONTEND=noninteractive -ARG NIFI_UID=1000 -ARG NIFI_GID=1000 -ARG UID=${NIFI_UID:-1000} -ARG GID=${NIFI_GID:-1000} ARG TZ="Europe/London" -ARG NIFI_PYTHON_FRAMEWORK_SOURCE_DIRECTORY="/opt/nifi/nifi-current/python/framework" -ARG NIFI_PYTHON_EXTENSIONS_SOURCE_DIRECTORY_DEFAULT="/opt/nifi/nifi-current/python_extensions" -ARG NIFI_PYTHON_WORKING_DIRECTORY="/opt/nifi/user_scripts" ENV TZ=${TZ} -ENV NIFI_PYTHON_FRAMEWORK_SOURCE_DIRECTORY=${NIFI_PYTHON_FRAMEWORK_SOURCE_DIRECTORY} -ENV NIFI_PYTHON_EXTENSIONS_SOURCE_DIRECTORY_DEFAULT=${NIFI_PYTHON_EXTENSIONS_SOURCE_DIRECTORY_DEFAULT} -ENV NIFI_PYTHON_WORKING_DIRECTORY=${NIFI_PYTHON_WORKING_DIRECTORY} -ENV PYTHONPATH=${NIFI_PYTHON_FRAMEWORK_SOURCE_DIRECTORY} +ENV PYTHONPATH=${NIFI_HOME}/python/framework # python stuff ENV PIP_PREFER_BINARY=1 ENV PIP_DISABLE_PIP_VERSION_CHECK=1 ENV PIP_NO_CACHE_DIR=1 -# Enables Python to generate .pyc files in the container -ENV PYTHONDONTWRITEBYTECODE=0 # Turns off buffering for easier container logging ENV PYTHONUNBUFFERED=1 -# default env vars to prevent NiFi from running on HTTP -ENV NIFI_WEB_HTTP_PORT="" -ENV NIFI_WEB_HTTP_HOST="" - USER root # Install system dependencies using the repositories provided by the base image. @@ -109,19 +94,10 @@ RUN uv export \ && uv pip install \ --no-cache-dir \ --break-system-packages \ - --target=${NIFI_PYTHON_FRAMEWORK_SOURCE_DIRECTORY} \ + --target=${NIFI_HOME}/python/framework \ -r /tmp/cogstack_nifi/requirements.txt \ --index-url https://pypi.org/simple -# install the local cogstack_nifi package so nifi.user_scripts imports resolve -COPY nifi /tmp/cogstack_nifi/nifi -RUN uv pip install \ - --no-cache-dir \ - --break-system-packages \ - --no-deps \ - --target=${NIFI_PYTHON_FRAMEWORK_SOURCE_DIRECTORY} \ - /tmp/cogstack_nifi - ####################################################################################################### # solve groovy grape proxy issues, grape ignores the current environment's proxy settings @@ -129,12 +105,9 @@ ENV JAVA_OPTS="-Dhttp.proxyHost=${HTTP_PROXY} -Dhttps.proxyHost=${HTTPS_PROXY} - # Install Groovy -RUN mkdir -p /opt/nifi/groovy -WORKDIR /opt/nifi/groovy/ +WORKDIR /opt/nifi/groovy RUN set -eux; \ - mkdir -p /opt/nifi/groovy; \ - cd /opt/nifi/groovy; \ url1="https://archive.apache.org/dist/groovy/${GROOVY_VERSION}/distribution/apache-groovy-binary-${GROOVY_VERSION}.zip"; \ url2="https://repo.maven.apache.org/maven2/org/apache/groovy/apache-groovy-binary/${GROOVY_VERSION}/apache-groovy-binary-${GROOVY_VERSION}.zip"; \ curl -fsSL --retry 8 --retry-delay 3 --connect-timeout 20 -o groovy.zip "$url1" \ @@ -148,95 +121,95 @@ RUN $GROOVY_BIN/grape -V install org.apache.avro avro 1.12.0 # INSTALL NAR extensions WORKDIR /opt/nifi/nifi-current/lib/ -# Install additional NAR extensions. +# Apache's base image already provides the standard NARs; add only missing extensions. RUN set -eux; \ base="https://repo.maven.apache.org/maven2/org/apache/nifi"; \ nars=( \ - nifi-atlassian-nar \ - nifi-aws-nar \ - nifi-aws-service-api-nar \ - nifi-aws-kinesis-nar \ - nifi-azure-nar \ - nifi-azure-services-api-nar \ nifi-snowflake-processors-nar \ nifi-snowflake-services-nar \ nifi-snowflake-services-api-nar \ - nifi-smb-nar \ - nifi-slack-nar \ - nifi-solr-nar \ nifi-sql-reporting-nar \ nifi-parquet-nar \ - nifi-database-dialect-service-nar \ - nifi-jolt-nar \ nifi-iotdb-nar \ - nifi-kafka-to-s3-nar \ - nifi-kafka-service-api-nar \ - nifi-kafka-service-aws-nar \ - nifi-kafka-nar \ - nifi-kerberos-iaa-providers-nar \ - nifi-key-service-nar \ - nifi-iceberg-shared-nar \ - nifi-iceberg-service-api-nar \ - nifi-iceberg-services-api-nar \ - nifi-iceberg-processors-nar \ - nifi-iceberg-rest-catalog-nar \ - nifi-iceberg-azure-nar \ - nifi-iceberg-aws-nar \ - nifi-iceberg-gcs-nar \ - nifi-iceberg-parquet-writer-nar \ - nifi-hashicorp-vault-nar \ - nifi-hashicorp-vault-client-service-api-nar \ nifi-hadoop-libraries-nar \ nifi-hadoop-nar \ - nifi-hl7-nar \ - nifi-github-nar \ - nifi-grpc-nar \ - nifi-groovyx-nar \ - nifi-geohash-nar \ - nifi-gitlab-nar \ nifi-graph-nar \ nifi-other-graph-services-nar \ nifi-graph-client-service-api-nar \ nifi-couchbase-nar \ - nifi-framework-zookeeper-nar \ - nifi-framework-kubernetes-nar \ - nifi-framework-nar \ - nifi-flow-registry-client-nar \ nifi-media-nar \ - nifi-mqtt-nar \ - nifi-mongodb-nar \ nifi-neo4j-cypher-service-nar \ nifi-nar-provider-service-api-nar \ - nifi-prometheus-nar \ - nifi-opentelemetry-nar \ - nifi-protobuf-services-nar \ - nifi-py4j-nar \ - nifi-redis-service-api-nar \ ); \ for a in "${nars[@]}"; do \ url="${base}/${a}/${NIFI_VERSION}/${a}-${NIFI_VERSION}.nar"; \ echo "Downloading $a"; \ - if ! curl -fsSL \ + curl -fsSL \ --connect-timeout 20 \ --retry 5 \ --retry-connrefused \ --retry-delay 3 \ --retry-max-time 120 \ -o "${a}-${NIFI_VERSION}.nar" \ - "$url"; then \ - echo "WARNING: NAR unavailable: $a @ ${NIFI_VERSION}"; \ - rm -f "${a}-${NIFI_VERSION}.nar"; \ - fi; \ - done; \ - chown ${UID}:${GID} ./*.nar + "$url"; \ + done ######################################################################################################## -# copy configuration files -WORKDIR /opt/nifi/nifi-current/conf/ +# Baseline resources work without a repository checkout. Processor directories +# can still be overridden by writable, deployment-owned volumes. +COPY nifi/__init__.py /tmp/cogstack_nifi/nifi/__init__.py +COPY nifi/user_scripts /tmp/cogstack_nifi/nifi/user_scripts +COPY nifi/user_python_extensions /tmp/cogstack_nifi/nifi/user_python_extensions +RUN uv pip install --no-cache-dir --break-system-packages --no-deps \ + --target=${NIFI_HOME}/python/framework /tmp/cogstack_nifi -# finalize -WORKDIR /opt/nifi/nifi-current/ +ARG NIFI_UID=1000 +ARG NIFI_GID=1000 -# add log file -RUN touch logs/nifi-app.log +COPY --chown=${NIFI_UID}:${NIFI_GID} nifi/drivers /opt/nifi/drivers +COPY --chown=${NIFI_UID}:${NIFI_GID} nifi/user_schemas /opt/nifi/user_schemas +COPY --chown=${NIFI_UID}:${NIFI_GID} nifi/user_templates /opt/nifi/user_templates +COPY --chown=${NIFI_UID}:${NIFI_GID} nifi/user_scripts /opt/nifi/user_scripts +COPY --chown=${NIFI_UID}:${NIFI_GID} nifi/user_python_extensions /opt/nifi/nifi-current/python_extensions +COPY --chmod=755 nifi/docker/healthcheck.sh /opt/nifi/scripts/cogstack-healthcheck.sh + +# Keep Apache's configuration defaults and startup script. Deployment credentials +# and mutable configuration from the checkout never enter the image. +RUN sed -i \ + -e 's|^nifi.security.autoreload.enabled=.*|nifi.security.autoreload.enabled=true|' \ + -e 's|^nifi.flowcontroller.graceful.shutdown.period=.*|nifi.flowcontroller.graceful.shutdown.period=30 sec|' \ + "${NIFI_HOME}/conf/nifi.properties" \ + && sed -i 's|^graceful.shutdown.seconds=.*|graceful.shutdown.seconds=60|' "${NIFI_HOME}/conf/bootstrap.conf" \ + && cp -a "${NIFI_HOME}/conf" /tmp/cogstack_nifi/nifi/conf + +# Apache ships owner-only executables; remap them only for custom UID builds. +RUN set -eux; \ + if [ "$(id -g nifi)" != "${NIFI_GID}" ]; then groupmod -o -g "${NIFI_GID}" nifi; fi; \ + if [ "$(id -u nifi)" != "${NIFI_UID}" ]; then \ + usermod -o -u "${NIFI_UID}" nifi; \ + chown -R "${NIFI_UID}:${NIFI_GID}" "${NIFI_HOME}" "${NIFI_TOOLKIT_HOME}"; \ + fi; \ + mkdir -p "${NIFI_HOME}/run" "${NIFI_HOME}/work/python" \ + "${NIFI_HOME}/logs/user_scripts" "${NIFI_HOME}/state/user_scripts/db" \ + /opt/nifi/pipeline/flowfile-errors /data /opt/models; \ + rm -rf "${NIFI_HOME}/python/framework/nifi/user_scripts"; \ + ln -s /opt/nifi/user_scripts "${NIFI_HOME}/python/framework/nifi/user_scripts"; \ + ln -s /opt/nifi/drivers /tmp/cogstack_nifi/nifi/drivers; \ + ln -s /opt/nifi/user_schemas /tmp/cogstack_nifi/nifi/user_schemas; \ + ln -s /opt/nifi/user_templates /tmp/cogstack_nifi/nifi/user_templates; \ + chown -R "${NIFI_UID}:${NIFI_GID}" \ + "${NIFI_HOME}"/{conf,logs,run,work,database_repository,flowfile_repository,content_repository,provenance_repository,state,nar_extensions,python_extensions} \ + /home/nifi /opt/nifi/pipeline/flowfile-errors /data /opt/models + +ENV HOME=/home/nifi +ENV NIFI_WEB_HTTPS_HOST="0.0.0.0" +ENV PATH=/usr/local/bin:${JAVA_HOME}/bin:/usr/local/sbin:/usr/sbin:/usr/bin:/sbin:/bin +ENV UV_CACHE_DIR=/opt/nifi/nifi-current/work/python/uv-cache +ENV PYTHONPYCACHEPREFIX=/opt/nifi/nifi-current/work/python/pycache +ENV NIFI_USER_SCRIPT_LOGS_DIR=/opt/nifi/nifi-current/logs/user_scripts +ENV NIFI_USER_SCRIPT_DB_DIR=/opt/nifi/nifi-current/state/user_scripts/db +WORKDIR /opt/nifi/nifi-current +USER ${NIFI_UID}:${NIFI_GID} +HEALTHCHECK --interval=30s --timeout=10s --start-period=300s --retries=3 \ + CMD ["/opt/nifi/scripts/cogstack-healthcheck.sh", "ready"] diff --git a/nifi/conf/nifi.properties b/nifi/conf/nifi.properties index 7d2277ba7..b189b0104 100644 --- a/nifi/conf/nifi.properties +++ b/nifi/conf/nifi.properties @@ -21,7 +21,7 @@ nifi.flow.configuration.archive.max.time=30 minutes nifi.flow.configuration.archive.max.storage=32 GB nifi.flow.configuration.archive.max.count=10 nifi.flowcontroller.autoResumeState=true -nifi.flowcontroller.graceful.shutdown.period=10 sec +nifi.flowcontroller.graceful.shutdown.period=30 sec nifi.flowservice.writedelay.interval=500 ms nifi.administrative.yield.duration=30 sec # If a component has no work to do (is "bored"), how long should we wait before checking again for work? @@ -48,7 +48,7 @@ nifi.templates.enabled=true nifi.python.command=python3.11 nifi.python.framework.source.directory=/opt/nifi/nifi-current/python/framework nifi.python.extensions.source.directory.default=/opt/nifi/nifi-current/python_extensions -nifi.python.working.directory=/opt/nifi/user_scripts +nifi.python.working.directory=/opt/nifi/nifi-current/work/python nifi.python.logs.directory=./logs nifi.python.max.processes.per.extension.type=10 nifi.python.max.processes=100 @@ -173,7 +173,7 @@ nifi.web.http.network.interface.default= ############################################# -nifi.web.https.host=nifi +nifi.web.https.host=0.0.0.0 nifi.web.https.port=8443 nifi.web.https.network.interface.default= nifi.web.jetty.working.directory=./work/jetty @@ -182,7 +182,7 @@ nifi.web.max.header.size=32 KB # IMPORTANT : be careful with this setting, it is buggy and single '/' don't work # After testing it seems that when running nifi in Docker you should set it via the NIFI_WEB_PROXY_CONTEXT_PATH ENV var, otherwise it wont work (needs a bug report) nifi.web.proxy.context.path=/nifi -nifi.web.proxy.host=localhost:8443 +nifi.web.proxy.host=localhost:8443,127.0.0.1:8443 nifi.web.max.content.size= nifi.web.max.requests.per.second=30000 nifi.web.max.access.token.requests.per.second=25 @@ -200,14 +200,14 @@ nifi.sensitive.props.key.protected= nifi.sensitive.props.algorithm=NIFI_PBKDF2_AES_GCM_256 nifi.sensitive.props.additional.keys= -nifi.security.autoreload.enabled=false +nifi.security.autoreload.enabled=true nifi.security.autoreload.interval=10 secs nifi.security.keystore=/security/certificates/nifi/nifi-keystore.jks -nifi.security.keystoreType=jks +nifi.security.keystoreType=JKS nifi.security.keystorePasswd=cogstackNifi nifi.security.keyPasswd=cogstackNifi nifi.security.truststore=/security/certificates/nifi/nifi-truststore.jks -nifi.security.truststoreType=jks +nifi.security.truststoreType=JKS nifi.security.truststorePasswd=cogstackNifi nifi.security.user.authorizer=single-user-authorizer nifi.security.allow.anonymous.authentication=false diff --git a/nifi/docker/healthcheck.sh b/nifi/docker/healthcheck.sh new file mode 100644 index 000000000..ae65264e2 --- /dev/null +++ b/nifi/docker/healthcheck.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +set -euo pipefail + +case "${1:-ready}" in + live) + exec "${NIFI_HOME:-/opt/nifi/nifi-current}/bin/nifi.sh" status >/dev/null + ;; + startup|ready) + host="${NIFI_HEALTHCHECK_HOST:-localhost}" + port="${NIFI_WEB_HTTPS_PORT:-${NIFI_INTERNAL_PORT:-8443}}" + exec curl --silent --show-error --fail --max-time "${NIFI_HEALTHCHECK_TIMEOUT:-5}" \ + --noproxy '*' --resolve "${host}:${port}:127.0.0.1" \ + --cacert "${NIFI_HEALTHCHECK_CA_FILE:-/security/certificates/root/root-ca.pem}" \ + --output /dev/null "https://${host}:${port}/nifi-api/authentication/configuration" + ;; + *) echo 'Usage: healthcheck.sh live|startup|ready' >&2; exit 2 ;; +esac diff --git a/scripts/tests/smoke_nifi_services.sh b/scripts/tests/smoke_nifi_services.sh index e105a7390..38b951b23 100755 --- a/scripts/tests/smoke_nifi_services.sh +++ b/scripts/tests/smoke_nifi_services.sh @@ -20,7 +20,7 @@ fi # shellcheck disable=SC1090 source "$SMOKE_HELPERS" -# Match nginx's canonical loopback host; localhost redirects to 127.0.0.1. +# Use loopback directly for browsers affected by localhost cookie handling. HOST="${NIFI_SMOKE_HOST:-127.0.0.1}" NIFI_PORT="${NIFI_EXTERNAL_PORT_NGINX:-8443}" # The sourced smoke helper reads this shared variable when checks run. @@ -108,6 +108,16 @@ fi echo "Running smoke checks against NiFi and nginx." wait_for_checks "NiFi" "$RETRIES" "$DELAY_SECONDS" "${SMOKE_CHECKS[@]}" +if [[ "$START_SERVICES" != "0" ]]; then + echo "Checking non-root users and container health probes." + for check in \ + "nifi|/opt/nifi/scripts/cogstack-healthcheck.sh" \ + "nifi-nginx|/usr/local/bin/cogstack-nginx-healthcheck.sh"; do + docker compose -f "${ROOT_DIR}/deploy/services.dev.yml" exec -T "${check%%|*}" sh -ec \ + 'test "$(id -u)" -ne 0; "$1" live; "$1" ready' -- "${check#*|}" + done +fi + echo "Checking authenticated NiFi API access." for attempt in $(seq 1 "$RETRIES"); do if check_authenticated_nifi_api; then diff --git a/scripts/tests/test_nifi_runtime.py b/scripts/tests/test_nifi_runtime.py new file mode 100644 index 000000000..1b3c1b65b --- /dev/null +++ b/scripts/tests/test_nifi_runtime.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +"""Check NiFi's upstream startup settings, persistence and credential isolation.""" + +import json +import os +from pathlib import Path +import shutil +import subprocess +import unittest + +ROOT = Path(__file__).resolve().parents[2] + + +@unittest.skipUnless(shutil.which("docker"), "Docker Compose is required") +class NiFiComposeRuntimeTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.models = {} + environment = {key: value for key, value in os.environ.items() if not key.startswith("COGSTACK_")} + for name in ("services.yml", "services.dev.yml"): + result = subprocess.run([ + "bash", "-c", 'source "$1" >/dev/null && docker compose -f "$2" config --format json', + "nifi-runtime-model", str(ROOT / "deploy/export_env_vars.sh"), str(ROOT / "deploy" / name), + ], env=environment, cwd=ROOT, capture_output=True, text=True, timeout=30, check=True) + cls.models[name] = json.loads(result.stdout)["services"] + + def test_proxy_does_not_receive_nifi_or_backend_login_credentials(self): + forbidden = {"NIFI_SINGLE_USER_CREDENTIALS_USERNAME", "NIFI_SINGLE_USER_CREDENTIALS_PASSWORD", + "SINGLE_USER_CREDENTIALS_USERNAME", "SINGLE_USER_CREDENTIALS_PASSWORD", + "NIFI_KEYSTORE_PASSWORD", "NIFI_TRUSTSTORE_PASSWORD", "NIFI_SENSITIVE_PROPS_KEY", + "KEYSTORE_PASSWORD", "TRUSTSTORE_PASSWORD", + "ELASTIC_PASSWORD", "DATABASE_PASSWORD", "MSSQL_SA_PASSWORD"} + for name, services in self.models.items(): + with self.subTest(file=name): + self.assertTrue(forbidden.isdisjoint(services["nifi-nginx"]["environment"])) + + def test_existing_deployment_settings_reach_apache_startup(self): + for name, services in self.models.items(): + settings = services["nifi"]["environment"] + with self.subTest(file=name): + for native, existing in ( + ("SINGLE_USER_CREDENTIALS_USERNAME", "NIFI_SINGLE_USER_CREDENTIALS_USERNAME"), + ("SINGLE_USER_CREDENTIALS_PASSWORD", "NIFI_SINGLE_USER_CREDENTIALS_PASSWORD"), + ("KEYSTORE_PATH", "NIFI_KEYSTORE_PATH"), ("KEYSTORE_TYPE", "NIFI_KEYSTORE_TYPE"), + ("KEYSTORE_PASSWORD", "NIFI_KEYSTORE_PASSWORD"), + ("TRUSTSTORE_PATH", "NIFI_TRUSTSTORE_PATH"), ("TRUSTSTORE_TYPE", "NIFI_TRUSTSTORE_TYPE"), + ("TRUSTSTORE_PASSWORD", "NIFI_TRUSTSTORE_PASSWORD"), + ("NIFI_WEB_HTTPS_PORT", "NIFI_INTERNAL_PORT"), + ("NIFI_REMOTE_INPUT_SOCKET_PORT", "NIFI_INPUT_SOCKET_PORT"), + ): + self.assertEqual(settings[native], settings[existing], native) + self.assertEqual(settings["AUTH"], "tls") + self.assertEqual(settings["NIFI_SECURITY_USER_AUTHORIZER"], "single-user-authorizer") + self.assertEqual(settings["NIFI_SECURITY_USER_LOGIN_IDENTITY_PROVIDER"], "single-user-provider") + + def test_development_conf_remains_persistent_and_writable(self): + nifi = self.models["services.dev.yml"]["nifi"] + runtime_conf = next(mount for mount in nifi["volumes"] if mount["target"] == "/opt/nifi/nifi-current/conf") + self.assertEqual(runtime_conf["type"], "volume") + self.assertFalse(runtime_conf.get("read_only", False)) + self.assertTrue(self.models["services.dev.yml"]["nifi-nginx"]["read_only"]) + + def test_python_generated_work_and_script_databases_are_outside_source(self): + for name, services in self.models.items(): + with self.subTest(file=name): + settings = services["nifi"]["environment"] + self.assertEqual(settings["NIFI_PYTHON_WORKING_DIRECTORY"], "/opt/nifi/nifi-current/work/python") + self.assertTrue(settings["NIFI_USER_SCRIPT_DB_DIR"].startswith("/opt/nifi/nifi-current/state/")) + self.assertTrue(settings["NIFI_USER_SCRIPT_LOGS_DIR"].startswith("/opt/nifi/nifi-current/logs/")) + + +if __name__ == "__main__": + unittest.main() diff --git a/security/scripts/nifi_init_create_user_auth.sh b/security/scripts/nifi_init_create_user_auth.sh index e06086bcc..f30fbce1e 100755 --- a/security/scripts/nifi_init_create_user_auth.sh +++ b/security/scripts/nifi_init_create_user_auth.sh @@ -3,10 +3,9 @@ # ============================================================================== # 🔐 Initialize NiFi Single-User Authentication (from host) # -# This script: -# 1. Starts the `cogstack-nifi` container (detached) -# 2. Executes the single-user credential setup script inside the container -# 3. Deletes the container afterward +# Compatibility helper for older published images. Rebuilt images apply the +# configured credentials automatically at startup. This helper only updates and +# restarts the running NiFi service; it never deletes NiFi or proxy containers. # # ⚠️ Assumes that the following environment variables are injected into # the container via `users_nifi.env` or equivalent: @@ -28,32 +27,8 @@ DEPLOY_DIR="${REPO_ROOT}/deploy" # shellcheck disable=SC1091 source "${DEPLOY_DIR}/export_env_vars.sh" -remove_existing_nifi_containers() { - local container_id - local -a container_ids=() - - while IFS= read -r container_id; do - if [[ -n "$container_id" ]]; then - container_ids+=("$container_id") - fi - done < <(docker ps -a -q --filter name="cogstack-nifi*") - - if (( ${#container_ids[@]} > 0 )); then - docker container rm -f "${container_ids[@]}" - fi -} - -echo "🧹 Removing any existing 'cogstack-nifi' container..." -remove_existing_nifi_containers - -echo "🚀 Starting NiFi container (for credential injection)..." -docker compose --project-directory "${DEPLOY_DIR}" -f "${DEPLOY_DIR}/services.yml" up -d nifi - -echo "🔐 Setting NiFi single-user credentials from inside the container..." -docker exec cogstack-nifi \ - /bin/bash /opt/nifi/nifi-current/security_scripts/nifi_create_single_user_auth.sh - -echo "🧼 Removing temporary NiFi container..." -remove_existing_nifi_containers - -echo "✅ NiFi single-user credential setup completed." +compose=(docker compose --project-directory "$DEPLOY_DIR" -f "$DEPLOY_DIR/services.yml") +"${compose[@]}" exec -T nifi /bin/bash -ec \ + '"${NIFI_HOME}/bin/nifi.sh" set-single-user-credentials "${NIFI_SINGLE_USER_CREDENTIALS_USERNAME:?}" "${NIFI_SINGLE_USER_CREDENTIALS_PASSWORD:?}"' +"${compose[@]}" restart nifi +echo "NiFi credentials updated. Rebuilt images apply these settings automatically." diff --git a/services/nginx/Dockerfile b/services/nginx/Dockerfile index 162b08d59..2175f3e0b 100644 --- a/services/nginx/Dockerfile +++ b/services/nginx/Dockerfile @@ -1,9 +1,25 @@ FROM nginx@sha256:600f7cd45e27aa13e9304c834ab7c1b0ff39df85ee1dcc5087955455d77bc83b -# Add configuration -RUN rm /etc/nginx/conf.d/default.conf -RUN rm /etc/nginx/nginx.conf +ARG NGINX_UID=1000 +ARG NGINX_GID=1000 -# Add site -RUN rm -rf /etc/nginx/sites-enabled -RUN mkdir /etc/nginx/sites-enabled/ +RUN set -eux; \ + apt-get update; \ + apt-get install -y --no-install-recommends curl; \ + rm -rf /var/lib/apt/lists/*; \ + rm -f /etc/nginx/conf.d/default.conf; \ + groupmod -o -g "${NGINX_GID}" nginx; \ + usermod -o -u "${NGINX_UID}" nginx; \ + mkdir -p /etc/nginx/sites-enabled + +COPY config/nginx.conf.template /etc/nginx/templates/nginx.conf.template +COPY --chmod=755 docker-entrypoint.sh /usr/local/bin/cogstack-nginx-entrypoint.sh +COPY --chmod=755 healthcheck.sh /usr/local/bin/cogstack-nginx-healthcheck.sh + +ENV HOME=/tmp +USER ${NGINX_UID}:${NGINX_GID} +ENTRYPOINT ["/usr/local/bin/cogstack-nginx-entrypoint.sh"] +CMD [] +STOPSIGNAL SIGQUIT +HEALTHCHECK --interval=30s --timeout=10s --start-period=30s --retries=3 \ + CMD ["/usr/local/bin/cogstack-nginx-healthcheck.sh", "ready"] diff --git a/services/nginx/config/nginx.conf.template b/services/nginx/config/nginx.conf.template index 305e717ad..ea7d4d90f 100644 --- a/services/nginx/config/nginx.conf.template +++ b/services/nginx/config/nginx.conf.template @@ -1,185 +1,81 @@ -user nginx; -pid /var/run/nginx.pid; - -worker_processes 1; +pid ${NGINX_RUNTIME_DIR}/nginx.pid; +worker_processes ${NGINX_WORKER_PROCESSES}; +error_log /dev/stderr notice; events { - worker_connections 1024; - } + worker_connections 1024; +} http { - - ## - # Basic Settings - ## - - sendfile on; - tcp_nopush on; - tcp_nodelay on; - keepalive_timeout 65; - types_hash_max_size 8192; - - # Hide nginx version information. + include /etc/nginx/mime.types; + default_type application/octet-stream; + access_log /dev/stdout; server_tokens off; - server_name_in_redirect off; - - # Specify MIME types for files. - include /etc/nginx/mime.types; - default_type application/octet-stream; - - #upstream elasticsearch { - # server elasticsearch-1:9200; - #} - - #upstream kibana { - # server kibana:5601; - #} - + sendfile on; + tcp_nopush on; + tcp_nodelay on; + keepalive_timeout 65; + types_hash_max_size 8192; + client_body_temp_path ${NGINX_RUNTIME_DIR}/client_temp; + proxy_temp_path ${NGINX_RUNTIME_DIR}/proxy_temp; + fastcgi_temp_path ${NGINX_RUNTIME_DIR}/fastcgi_temp; + uwsgi_temp_path ${NGINX_RUNTIME_DIR}/uwsgi_temp; + scgi_temp_path ${NGINX_RUNTIME_DIR}/scgi_temp; + + resolver ${NGINX_DNS_RESOLVER} valid=10s; + resolver_timeout 3s; upstream nifi { - server nifi:${NIFI_INTERNAL_PORT}; + zone nifi 64k; + server ${NIFI_UPSTREAM_HOST}:${NIFI_UPSTREAM_PORT} resolve; } + proxy_ssl_server_name on; + proxy_ssl_name ${NIFI_UPSTREAM_TLS_NAME}; + proxy_ssl_trusted_certificate "${NGINX_UPSTREAM_CA}"; + proxy_ssl_verify on; + proxy_ssl_verify_depth 3; + proxy_ssl_protocols TLSv1.2 TLSv1.3; include /etc/nginx/sites-enabled/*.conf; - #server { - # listen 9200; - # server_name elasticsearch.local; - - # location / { - # proxy_pass http://elasticsearch; - # - # proxy_set_header X-ProxyScheme http; - # proxy_set_header X-ProxyHost nginx.docker.localhost; - # proxy_set_header X-ProxyPort 9200; - # proxy_set_header X-ProxyContextPath /; - # } - #} - - #server { - # listen 5601; - # server_name kibana.local; - - # location / { - # proxy_pass http://kibana; - # - # proxy_set_header X-ProxyScheme http; - # proxy_set_header X-ProxyHost nginx.docker.localhost; - # proxy_set_header X-ProxyPort 5601; - # proxy_set_header X-ProxyContextPath /; - # } - #} - - #server { - # listen 80; - # listen [::]:80; - # server_name nginx.local; - # return 301 https://$host$request_uri; - #} - + # Local liveness does not depend on NiFi or any external service. server { - listen ${NIFI_INTERNAL_PORT_NGINX} ssl; - server_name ${NGINX_NIFI_SERVER_NAME}; - - proxy_ssl_server_name on; - - # Firefox has trouble with the localhost-scoped NiFi auth cookie. - # Redirect localhost to 127.0.0.1 so the browser gets a stable IP-scoped - # cookie domain instead. - if ($host = localhost) { - return 308 https://127.0.0.1:8443$request_uri; + listen 127.0.0.1:${NGINX_HEALTH_PORT}; + access_log off; + location = /healthz/live { + default_type text/plain; + return 200 'ok\n'; } - - ssl_certificate /certificates/nifi/nifi.pem; - ssl_certificate_key /certificates/nifi/nifi.key; - ssl_client_certificate /certificates/root/root-ca.pem; - ssl_trusted_certificate /certificates/root/root-ca.pem; - - # if you comment this out then you will automatically enable authentication, because nginx won't certify your session... - - #proxy_ssl_certificate /certificates/nifi/nifi.pem; - #proxy_ssl_certificate_key /certificates/nifi/nifi.key; - #proxy_ssl_trusted_certificate /certificates/nifi/nifi.pem; - - - location / { - proxy_set_header Host nifi; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-ProxyHost $host; - proxy_set_header X-ProxyPort ${NIFI_WEB_PROXY_PORT}; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-ProxyScheme $scheme; - # proxy_set_header X-ProxyContextPath /; - # proxy_set_header X-Forwarded-Context /; - proxy_pass https://nifi; + location = /healthz/ready { + proxy_set_header Host ${NIFI_UPSTREAM_TLS_NAME}; + proxy_connect_timeout 3s; + proxy_read_timeout 3s; + proxy_pass https://nifi/nifi-api/authentication/configuration; } + } - location /nifi { - proxy_set_header Host nifi; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-ProxyHost $host; - proxy_set_header X-ProxyPort ${NIFI_WEB_PROXY_PORT}; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-ProxyScheme $scheme; - proxy_pass https://nifi; + server { + listen ${NIFI_INTERNAL_PORT_NGINX} ssl; + server_name ${NGINX_NIFI_SERVER_NAME}; + ssl_certificate "${NGINX_TLS_CERTIFICATE}"; + ssl_certificate_key "${NGINX_TLS_CERTIFICATE_KEY}"; + ssl_protocols TLSv1.2 TLSv1.3; + + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-ProxyHost $host; + proxy_set_header X-ProxyPort ${NIFI_WEB_PROXY_PORT}; + proxy_set_header X-ProxyScheme ${NGINX_PROXY_SCHEME}; + proxy_set_header X-ProxyContextPath "${NGINX_PROXY_CONTEXT_PATH}"; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto ${NGINX_PROXY_SCHEME}; + + location ${NGINX_PROXY_CONTEXT_PATH}/ { + proxy_pass https://nifi/; } - - location ^~ /nifi-api/ { - proxy_set_header Host nifi; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-ProxyHost $host; - proxy_set_header X-ProxyPort ${NIFI_WEB_PROXY_PORT}; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-ProxyScheme $scheme; + location ${NGINX_PROXY_CONTEXT_PATH}/nifi-api/ { proxy_hide_header X-Frame-Options; proxy_pass https://nifi/nifi-api/; } - - # location ^~ /nifi-api/flowfile-queues/ { - # proxy_set_header Host nifi; - # proxy_set_header X-Real-IP nifi; - # proxy_set_header X-ProxyHost $host; - # proxy_set_header X-ProxyPort ${NIFI_WEB_PROXY_PORT}; - # proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - # proxy_set_header X-ProxyScheme $scheme; - # proxy_hide_header Content-Security-Policy; - # add_header Content-Security-Policy ""; - # proxy_pass https://nifi/nifi-api/flowfile-queues/; - # } - - location ^~ /media/ { - proxy_set_header Host nifi; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-ProxyHost $host; - proxy_set_header X-ProxyPort ${NIFI_WEB_PROXY_PORT}; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-ProxyScheme $scheme; - proxy_pass https://nifi/media/; - } - - location ^~ /content-viewer/ { - proxy_set_header Host nifi; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-ProxyHost $host; - proxy_set_header X-ProxyPort ${NIFI_WEB_PROXY_PORT}; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-ProxyScheme $scheme; - proxy_pass https://nifi/content-viewer/; - } - - location ~ ^/nifi-standard-content-viewer-[0-9.]+/ { - proxy_set_header Host nifi; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-ProxyHost $host; - proxy_set_header X-ProxyPort ${NIFI_WEB_PROXY_PORT}; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-ProxyScheme $scheme; - - # Relax CSP if needed - # proxy_hide_header Content-Security-Policy; - # add_header Content-Security-Policy ""; - - proxy_pass https://nifi; - } } } diff --git a/services/nginx/docker-entrypoint.sh b/services/nginx/docker-entrypoint.sh new file mode 100644 index 000000000..f9e237b27 --- /dev/null +++ b/services/nginx/docker-entrypoint.sh @@ -0,0 +1,50 @@ +#!/bin/sh +set -eu +umask 077 + +if [ "$#" -gt 0 ]; then exec "$@"; fi + +: "${NIFI_UPSTREAM_HOST:=nifi}" +: "${NIFI_UPSTREAM_PORT:=${NIFI_INTERNAL_PORT:-8443}}" +: "${NIFI_UPSTREAM_TLS_NAME:=$NIFI_UPSTREAM_HOST}" +: "${NIFI_INTERNAL_PORT_NGINX:=8443}" +: "${NIFI_WEB_PROXY_PORT:=8443}" +: "${NGINX_NIFI_SERVER_NAME:=_}" +: "${NGINX_PROXY_CONTEXT_PATH:=}" +: "${NGINX_PROXY_SCHEME:=https}" +: "${NGINX_TLS_CERTIFICATE:=/certificates/nifi/nifi.pem}" +: "${NGINX_TLS_CERTIFICATE_KEY:=/certificates/nifi/nifi.key}" +: "${NGINX_UPSTREAM_CA:=/certificates/root/root-ca.pem}" +: "${NGINX_HEALTH_PORT:=8080}" +: "${NGINX_WORKER_PROCESSES:=1}" +: "${NGINX_RUNTIME_DIR:=/tmp/nginx}" +: "${NGINX_DNS_RESOLVER:=$(awk '/^nameserver[ \t]/{print $2; exit}' /etc/resolv.conf)}" +case "$NGINX_DNS_RESOLVER" in *:*|*.*) ;; *) echo 'Supply a DNS resolver address' >&2; exit 1 ;; esac +case "$NGINX_DNS_RESOLVER" in \[*\]) ;; *:* ) NGINX_DNS_RESOLVER="[$NGINX_DNS_RESOLVER]" ;; esac + +for port in "$NIFI_UPSTREAM_PORT" "$NIFI_INTERNAL_PORT_NGINX" "$NIFI_WEB_PROXY_PORT" "$NGINX_HEALTH_PORT"; do + case "$port" in ''|*[!0-9]*) echo 'Proxy ports must be numeric' >&2; exit 1 ;; esac + [ "$port" -gt 0 ] && [ "$port" -le 65535 ] || { echo 'Proxy port is out of range' >&2; exit 1; } +done +case "$NGINX_PROXY_SCHEME" in http|https) ;; *) echo 'Invalid NGINX_PROXY_SCHEME' >&2; exit 1 ;; esac +case "$NGINX_PROXY_CONTEXT_PATH" in /) NGINX_PROXY_CONTEXT_PATH='' ;; */) NGINX_PROXY_CONTEXT_PATH=${NGINX_PROXY_CONTEXT_PATH%/} ;; esac +for value in "$NIFI_UPSTREAM_HOST" "$NIFI_UPSTREAM_TLS_NAME" "$NGINX_NIFI_SERVER_NAME" \ + "$NGINX_PROXY_CONTEXT_PATH" "$NGINX_TLS_CERTIFICATE" "$NGINX_TLS_CERTIFICATE_KEY" \ + "$NGINX_UPSTREAM_CA" "$NGINX_RUNTIME_DIR" "$NGINX_WORKER_PROCESSES" "$NGINX_DNS_RESOLVER"; do + case "$value" in *';'*|*'{'*|*'}'*|*'"'*|*'\'*|*' +'*) echo 'Invalid character in proxy configuration' >&2; exit 1 ;; esac +done +case "$NGINX_PROXY_CONTEXT_PATH" in ''|/*) ;; *) echo 'Proxy context must start with /' >&2; exit 1 ;; esac +for certificate in "$NGINX_TLS_CERTIFICATE" "$NGINX_TLS_CERTIFICATE_KEY" "$NGINX_UPSTREAM_CA"; do + [ -f "$certificate" ] && [ -r "$certificate" ] || { echo "Certificate input is not readable by the runtime user: $certificate" >&2; exit 1; } +done +mkdir -p "$NGINX_RUNTIME_DIR/client_temp" "$NGINX_RUNTIME_DIR/proxy_temp" \ + "$NGINX_RUNTIME_DIR/fastcgi_temp" "$NGINX_RUNTIME_DIR/uwsgi_temp" "$NGINX_RUNTIME_DIR/scgi_temp" +export NIFI_UPSTREAM_HOST NIFI_UPSTREAM_PORT NIFI_UPSTREAM_TLS_NAME NIFI_INTERNAL_PORT_NGINX \ + NIFI_WEB_PROXY_PORT NGINX_NIFI_SERVER_NAME NGINX_PROXY_CONTEXT_PATH NGINX_PROXY_SCHEME \ + NGINX_TLS_CERTIFICATE NGINX_TLS_CERTIFICATE_KEY NGINX_UPSTREAM_CA NGINX_HEALTH_PORT \ + NGINX_WORKER_PROCESSES NGINX_RUNTIME_DIR NGINX_DNS_RESOLVER +envsubst '${NIFI_UPSTREAM_HOST} ${NIFI_UPSTREAM_PORT} ${NIFI_UPSTREAM_TLS_NAME} ${NIFI_INTERNAL_PORT_NGINX} ${NIFI_WEB_PROXY_PORT} ${NGINX_NIFI_SERVER_NAME} ${NGINX_PROXY_CONTEXT_PATH} ${NGINX_PROXY_SCHEME} ${NGINX_TLS_CERTIFICATE} ${NGINX_TLS_CERTIFICATE_KEY} ${NGINX_UPSTREAM_CA} ${NGINX_HEALTH_PORT} ${NGINX_WORKER_PROCESSES} ${NGINX_RUNTIME_DIR} ${NGINX_DNS_RESOLVER}' \ + < "${NGINX_CONFIG_TEMPLATE:-/etc/nginx/templates/nginx.conf.template}" > "$NGINX_RUNTIME_DIR/nginx.conf" +nginx -t -c "$NGINX_RUNTIME_DIR/nginx.conf" +exec nginx -c "$NGINX_RUNTIME_DIR/nginx.conf" -g 'daemon off;' diff --git a/services/nginx/healthcheck.sh b/services/nginx/healthcheck.sh new file mode 100644 index 000000000..ba8609ff1 --- /dev/null +++ b/services/nginx/healthcheck.sh @@ -0,0 +1,9 @@ +#!/bin/sh +set -eu +case "${1:-ready}" in + live) path=live ;; + startup|ready) path=ready ;; + *) echo 'Usage: healthcheck.sh live|startup|ready' >&2; exit 2 ;; +esac +exec curl --silent --show-error --fail --noproxy '*' --max-time "${NGINX_HEALTH_TIMEOUT:-5}" \ + --output /dev/null "http://127.0.0.1:${NGINX_HEALTH_PORT:-8080}/healthz/$path" From b22700490f8e17605117957a9fedb6cce74cf4cb Mon Sep 17 00:00:00 2001 From: vladd-bit Date: Tue, 10 Nov 2026 17:32:03 +0000 Subject: [PATCH 25/28] NiFi: updated templates. --- .../opensearch_ingest_docs_db_ocr_service_to_es.json | 2 +- nifi/user_templates/opensearch_ingest_docs_db_to_es.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nifi/user_templates/opensearch_ingest_docs_db_ocr_service_to_es.json b/nifi/user_templates/opensearch_ingest_docs_db_ocr_service_to_es.json index 5cb13c2e8..122f50fce 100644 --- a/nifi/user_templates/opensearch_ingest_docs_db_ocr_service_to_es.json +++ b/nifi/user_templates/opensearch_ingest_docs_db_ocr_service_to_es.json @@ -1 +1 @@ -{"flowContents":{"identifier":"af2b9237-8c32-355c-9495-3be0e06b4c25","instanceIdentifier":"74590095-019f-1000-9c76-52dada17636f","name":"opensearch_ingest_docs_db_ocr_service_to_es","comments":"","position":{"x":872.0,"y":352.0},"processGroups":[],"remoteProcessGroups":[],"processors":[{"identifier":"b0a7e1fd-0197-1000-7799-44654203e24f","instanceIdentifier":"c652a637-5daa-3016-69dc-eb043ae56375","name":"ExecuteSQLRecord","comments":"","position":{"x":-1152.0,"y":-1528.0},"type":"org.apache.nifi.processors.standard.ExecuteSQLRecord","bundle":{"group":"org.apache.nifi","artifact":"nifi-standard-nar","version":"2.10.0"},"properties":{"Set Auto Commit":"true","Max Wait Time":"0 seconds","SQL Query":null,"Normalize Table/Column Names":"false","Database Connection Pooling Service":"ae4d900f-d06e-3cc3-acc5-4f6f222a5064","Default Decimal Scale":"0","SQL Post-Query":null,"Fetch Size":"0","Output Batch Size":"0","Use Avro Logical Types":"true","Record Writer":"4ad0bf71-a5de-39f7-9f74-07213d115a9f","SQL Pre-Query":null,"Default Decimal Precision":"10","Max Rows Per FlowFile":"1"},"propertyDescriptors":{"Set Auto Commit":{"name":"Set Auto Commit","displayName":"Set Auto Commit","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Max Wait Time":{"name":"Max Wait Time","displayName":"Max Wait Time","identifiesControllerService":false,"sensitive":false,"dynamic":false},"SQL Query":{"name":"SQL Query","displayName":"SQL Query","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Normalize Table/Column Names":{"name":"Normalize Table/Column Names","displayName":"Normalize Table/Column Names","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Database Connection Pooling Service":{"name":"Database Connection Pooling Service","displayName":"Database Connection Pooling Service","identifiesControllerService":true,"sensitive":false,"dynamic":false},"Default Decimal Scale":{"name":"Default Decimal Scale","displayName":"Default Decimal Scale","identifiesControllerService":false,"sensitive":false,"dynamic":false},"SQL Post-Query":{"name":"SQL Post-Query","displayName":"SQL Post-Query","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Fetch Size":{"name":"Fetch Size","displayName":"Fetch Size","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Output Batch Size":{"name":"Output Batch Size","displayName":"Output Batch Size","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Use Avro Logical Types":{"name":"Use Avro Logical Types","displayName":"Use Avro Logical Types","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Record Writer":{"name":"Record Writer","displayName":"Record Writer","identifiesControllerService":true,"sensitive":false,"dynamic":false},"SQL Pre-Query":{"name":"SQL Pre-Query","displayName":"SQL Pre-Query","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Default Decimal Precision":{"name":"Default Decimal Precision","displayName":"Default Decimal Precision","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Max Rows Per FlowFile":{"name":"Max Rows Per FlowFile","displayName":"Max Rows Per FlowFile","identifiesControllerService":false,"sensitive":false,"dynamic":false}},"style":{},"schedulingPeriod":"0 sec","schedulingStrategy":"TIMER_DRIVEN","executionNode":"ALL","penaltyDuration":"30 sec","yieldDuration":"1 sec","bulletinLevel":"WARN","runDurationMillis":0,"concurrentlySchedulableTaskCount":1,"autoTerminatedRelationships":[],"scheduledState":"ENABLED","retryCount":10,"retriedRelationships":[],"backoffMechanism":"PENALIZE_FLOWFILE","maxBackoffPeriod":"10 mins","componentType":"PROCESSOR","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"63d97939-57b6-36a3-91e2-ee0f4cee2b02","instanceIdentifier":"b37a4cb7-8b0d-3a5b-4333-df1160bc8967","name":"SplitJson","comments":"","position":{"x":-1152.0,"y":-688.0},"type":"org.apache.nifi.processors.standard.SplitJson","bundle":{"group":"org.apache.nifi","artifact":"nifi-standard-nar","version":"2.10.0"},"properties":{"Max String Length":"20 MB","Null Value Representation":"empty string","JsonPath Expression":"$.*"},"propertyDescriptors":{"Max String Length":{"name":"Max String Length","displayName":"Max String Length","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Null Value Representation":{"name":"Null Value Representation","displayName":"Null Value Representation","identifiesControllerService":false,"sensitive":false,"dynamic":false},"JsonPath Expression":{"name":"JsonPath Expression","displayName":"JsonPath Expression","identifiesControllerService":false,"sensitive":false,"dynamic":false}},"style":{},"schedulingPeriod":"0 sec","schedulingStrategy":"TIMER_DRIVEN","executionNode":"ALL","penaltyDuration":"30 sec","yieldDuration":"1 sec","bulletinLevel":"WARN","runDurationMillis":0,"concurrentlySchedulableTaskCount":1,"autoTerminatedRelationships":["original"],"scheduledState":"ENABLED","retryCount":10,"retriedRelationships":[],"backoffMechanism":"PENALIZE_FLOWFILE","maxBackoffPeriod":"10 mins","componentType":"PROCESSOR","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"b0a7e1fe-0197-1000-7d26-f460ed2eec5e","instanceIdentifier":"6ddb9b95-fb84-34e6-c230-f4047af5dee9","name":"GenerateTableFetch-encounters","comments":"","position":{"x":-1152.0,"y":-1720.0},"type":"org.apache.nifi.processors.standard.GenerateTableFetch","bundle":{"group":"org.apache.nifi","artifact":"nifi-standard-nar","version":"2.10.0"},"properties":{"Table Name":"encounters","Max Wait Time":"0 seconds","Partition Size":"1000","Database Connection Pooling Service":"ae4d900f-d06e-3cc3-acc5-4f6f222a5064","Database Type":"PostgreSQL","Columns to Return":null,"Maximum-value Columns":"cid","Custom ORDER BY Column":null,"Output Empty FlowFile on Zero Results":"false","Database Dialect Service":null,"Column for Value Partitioning":"cid","Additional WHERE Clause":null},"propertyDescriptors":{"Table Name":{"name":"Table Name","displayName":"Table Name","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Max Wait Time":{"name":"Max Wait Time","displayName":"Max Wait Time","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Partition Size":{"name":"Partition Size","displayName":"Partition Size","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Database Connection Pooling Service":{"name":"Database Connection Pooling Service","displayName":"Database Connection Pooling Service","identifiesControllerService":true,"sensitive":false,"dynamic":false},"Database Type":{"name":"Database Type","displayName":"Database Type","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Columns to Return":{"name":"Columns to Return","displayName":"Columns to Return","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Maximum-value Columns":{"name":"Maximum-value Columns","displayName":"Maximum-value Columns","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Custom ORDER BY Column":{"name":"Custom ORDER BY Column","displayName":"Custom ORDER BY Column","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Output Empty FlowFile on Zero Results":{"name":"Output Empty FlowFile on Zero Results","displayName":"Output Empty FlowFile on Zero Results","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Database Dialect Service":{"name":"Database Dialect Service","displayName":"Database Dialect Service","identifiesControllerService":true,"sensitive":false,"dynamic":false},"Column for Value Partitioning":{"name":"Column for Value Partitioning","displayName":"Column for Value Partitioning","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Additional WHERE Clause":{"name":"Additional WHERE Clause","displayName":"Additional WHERE Clause","identifiesControllerService":false,"sensitive":false,"dynamic":false}},"style":{},"schedulingPeriod":"1 sec","schedulingStrategy":"TIMER_DRIVEN","executionNode":"ALL","penaltyDuration":"30 sec","yieldDuration":"1 sec","bulletinLevel":"WARN","runDurationMillis":0,"concurrentlySchedulableTaskCount":1,"autoTerminatedRelationships":[],"scheduledState":"ENABLED","retryCount":10,"retriedRelationships":[],"backoffMechanism":"PENALIZE_FLOWFILE","maxBackoffPeriod":"10 mins","componentType":"PROCESSOR","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"25d6775d-c9fa-3166-9203-973e735e8877","instanceIdentifier":"74d72c6e-019f-1000-bb5f-e95c500a61f5","name":"CogStackParseCogStackServiceResult","comments":"","position":{"x":-1152.0,"y":-896.0},"type":"CogStackParseCogStackServiceResult","bundle":{"group":"org.apache.nifi","artifact":"python-extensions","version":"0.0.3"},"properties":{"medcat_empty_text_handling":"drop","document_text_field_name":"text","medcat_no_annotations_handling":"restore","medcat_output_mode":"not_set","service_message_type":"ocr","output_text_field_name":"text","document_id_field_name":"id","medcat_deid_keep_annotations":"true"},"propertyDescriptors":{"medcat_empty_text_handling":{"name":"medcat_empty_text_handling","displayName":"medcat_empty_text_handling","identifiesControllerService":false,"sensitive":false,"dynamic":false},"document_text_field_name":{"name":"document_text_field_name","displayName":"document_text_field_name","identifiesControllerService":false,"sensitive":false,"dynamic":false},"medcat_no_annotations_handling":{"name":"medcat_no_annotations_handling","displayName":"medcat_no_annotations_handling","identifiesControllerService":false,"sensitive":false,"dynamic":false},"medcat_output_mode":{"name":"medcat_output_mode","displayName":"medcat_output_mode","identifiesControllerService":false,"sensitive":false,"dynamic":false},"service_message_type":{"name":"service_message_type","displayName":"service_message_type","identifiesControllerService":false,"sensitive":false,"dynamic":false},"output_text_field_name":{"name":"output_text_field_name","displayName":"output_text_field_name","identifiesControllerService":false,"sensitive":false,"dynamic":false},"document_id_field_name":{"name":"document_id_field_name","displayName":"document_id_field_name","identifiesControllerService":false,"sensitive":false,"dynamic":false},"medcat_deid_keep_annotations":{"name":"medcat_deid_keep_annotations","displayName":"medcat_deid_keep_annotations","identifiesControllerService":false,"sensitive":false,"dynamic":false}},"style":{},"schedulingPeriod":"0 sec","schedulingStrategy":"TIMER_DRIVEN","executionNode":"ALL","penaltyDuration":"30 sec","yieldDuration":"1 sec","bulletinLevel":"WARN","runDurationMillis":25,"concurrentlySchedulableTaskCount":1,"autoTerminatedRelationships":["original"],"scheduledState":"ENABLED","retryCount":10,"retriedRelationships":[],"backoffMechanism":"PENALIZE_FLOWFILE","maxBackoffPeriod":"10 mins","componentType":"PROCESSOR","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"c1de59a6-f215-3d9b-8597-237a1a50852d","instanceIdentifier":"4c3e5f3c-eb76-3f7f-3714-792ddd0cfb3a","name":"EvaluateJsonPath","comments":"","position":{"x":-496.0,"y":-688.0},"type":"org.apache.nifi.processors.standard.EvaluateJsonPath","bundle":{"group":"org.apache.nifi","artifact":"nifi-standard-nar","version":"2.10.0"},"properties":{"Destination":"flowfile-attribute","Max String Length":"20 MB","Return Type":"auto-detect","Null Value Representation":"empty string","id":"$.id","Path Not Found Behavior":"ignore"},"propertyDescriptors":{"Destination":{"name":"Destination","displayName":"Destination","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Max String Length":{"name":"Max String Length","displayName":"Max String Length","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Return Type":{"name":"Return Type","displayName":"Return Type","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Null Value Representation":{"name":"Null Value Representation","displayName":"Null Value Representation","identifiesControllerService":false,"sensitive":false,"dynamic":false},"id":{"name":"id","displayName":"id","identifiesControllerService":false,"sensitive":false,"dynamic":true},"Path Not Found Behavior":{"name":"Path Not Found Behavior","displayName":"Path Not Found Behavior","identifiesControllerService":false,"sensitive":false,"dynamic":false}},"style":{},"schedulingPeriod":"0 sec","schedulingStrategy":"TIMER_DRIVEN","executionNode":"ALL","penaltyDuration":"30 sec","yieldDuration":"1 sec","bulletinLevel":"WARN","runDurationMillis":0,"concurrentlySchedulableTaskCount":1,"autoTerminatedRelationships":[],"scheduledState":"ENABLED","retryCount":10,"retriedRelationships":[],"backoffMechanism":"PENALIZE_FLOWFILE","maxBackoffPeriod":"10 mins","componentType":"PROCESSOR","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"22e4a99b-9b48-3dea-920e-e1600becc662","instanceIdentifier":"fb359fe1-f5a9-39b5-94e9-c8dec6112a99","name":"InvokeHTTP","comments":"","position":{"x":-1152.0,"y":-1120.0},"type":"org.apache.nifi.processors.standard.InvokeHTTP","bundle":{"group":"org.apache.nifi","artifact":"nifi-standard-nar","version":"2.10.0"},"properties":{"Request Content-Encoding":"DISABLED","Request Multipart Form-Data Filename Enabled":"true","Request Chunked Transfer-Encoding Enabled":"false","Response Header Request Attributes Prefix":null,"Proxy Configuration Service":null,"HTTP/2 Disabled":"False","Connection Timeout":"15 secs","Response Cookie Strategy":"DISABLED","Socket Read Timeout":"30 secs","Socket Idle Connections":"5","Request Body Enabled":"true","OAuth2 Access Token Refresh Strategy":"ON_TOKEN_EXPIRATION","HTTP URL":"http://ocr-service:8090/api/process","Request OAuth2 Access Token Provider":null,"Socket Idle Timeout":"5 mins","Response Redirects Enabled":"True","Socket Write Timeout":"30 secs","Request Header Attributes Pattern":null,"Response FlowFile Naming Strategy":"RANDOM","Response Cache Enabled":"false","Request Date Header Enabled":"True","Request Failure Penalization Enabled":"false","Response Body Attribute Size":"256","SSL Context Service":null,"Response Generation Required":"false","Request User-Agent":null,"Response Header Request Attributes Enabled":"false","HTTP Method":"POST","Request Username":null,"Request Content-Type":"${mime.type}","Response Body Attribute Name":null,"Request Digest Authentication Enabled":"false","Request Multipart Form-Data Name":null,"Response Cache Size":"10MB","Response Body Ignored":"false"},"propertyDescriptors":{"Request Content-Encoding":{"name":"Request Content-Encoding","displayName":"Request Content-Encoding","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Request Multipart Form-Data Filename Enabled":{"name":"Request Multipart Form-Data Filename Enabled","displayName":"Request Multipart Form-Data Filename Enabled","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Request Chunked Transfer-Encoding Enabled":{"name":"Request Chunked Transfer-Encoding Enabled","displayName":"Request Chunked Transfer-Encoding Enabled","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Response Header Request Attributes Prefix":{"name":"Response Header Request Attributes Prefix","displayName":"Response Header Request Attributes Prefix","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Proxy Configuration Service":{"name":"Proxy Configuration Service","displayName":"Proxy Configuration Service","identifiesControllerService":true,"sensitive":false,"dynamic":false},"HTTP/2 Disabled":{"name":"HTTP/2 Disabled","displayName":"HTTP/2 Disabled","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Connection Timeout":{"name":"Connection Timeout","displayName":"Connection Timeout","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Response Cookie Strategy":{"name":"Response Cookie Strategy","displayName":"Response Cookie Strategy","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Request Password":{"name":"Request Password","displayName":"Request Password","identifiesControllerService":false,"sensitive":true,"dynamic":false},"Socket Read Timeout":{"name":"Socket Read Timeout","displayName":"Socket Read Timeout","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Socket Idle Connections":{"name":"Socket Idle Connections","displayName":"Socket Idle Connections","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Request Body Enabled":{"name":"Request Body Enabled","displayName":"Request Body Enabled","identifiesControllerService":false,"sensitive":false,"dynamic":false},"OAuth2 Access Token Refresh Strategy":{"name":"OAuth2 Access Token Refresh Strategy","displayName":"OAuth2 Access Token Refresh Strategy","identifiesControllerService":false,"sensitive":false,"dynamic":false},"HTTP URL":{"name":"HTTP URL","displayName":"HTTP URL","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Request OAuth2 Access Token Provider":{"name":"Request OAuth2 Access Token Provider","displayName":"Request OAuth2 Access Token Provider","identifiesControllerService":true,"sensitive":false,"dynamic":false},"Socket Idle Timeout":{"name":"Socket Idle Timeout","displayName":"Socket Idle Timeout","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Response Redirects Enabled":{"name":"Response Redirects Enabled","displayName":"Response Redirects Enabled","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Socket Write Timeout":{"name":"Socket Write Timeout","displayName":"Socket Write Timeout","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Request Header Attributes Pattern":{"name":"Request Header Attributes Pattern","displayName":"Request Header Attributes Pattern","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Response FlowFile Naming Strategy":{"name":"Response FlowFile Naming Strategy","displayName":"Response FlowFile Naming Strategy","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Response Cache Enabled":{"name":"Response Cache Enabled","displayName":"Response Cache Enabled","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Request Date Header Enabled":{"name":"Request Date Header Enabled","displayName":"Request Date Header Enabled","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Request Failure Penalization Enabled":{"name":"Request Failure Penalization Enabled","displayName":"Request Failure Penalization Enabled","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Response Body Attribute Size":{"name":"Response Body Attribute Size","displayName":"Response Body Attribute Size","identifiesControllerService":false,"sensitive":false,"dynamic":false},"SSL Context Service":{"name":"SSL Context Service","displayName":"SSL Context Service","identifiesControllerService":true,"sensitive":false,"dynamic":false},"Response Generation Required":{"name":"Response Generation Required","displayName":"Response Generation Required","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Request User-Agent":{"name":"Request User-Agent","displayName":"Request User-Agent","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Response Header Request Attributes Enabled":{"name":"Response Header Request Attributes Enabled","displayName":"Response Header Request Attributes Enabled","identifiesControllerService":false,"sensitive":false,"dynamic":false},"HTTP Method":{"name":"HTTP Method","displayName":"HTTP Method","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Request Username":{"name":"Request Username","displayName":"Request Username","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Request Content-Type":{"name":"Request Content-Type","displayName":"Request Content-Type","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Response Body Attribute Name":{"name":"Response Body Attribute Name","displayName":"Response Body Attribute Name","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Request Digest Authentication Enabled":{"name":"Request Digest Authentication Enabled","displayName":"Request Digest Authentication Enabled","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Request Multipart Form-Data Name":{"name":"Request Multipart Form-Data Name","displayName":"Request Multipart Form-Data Name","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Response Cache Size":{"name":"Response Cache Size","displayName":"Response Cache Size","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Response Body Ignored":{"name":"Response Body Ignored","displayName":"Response Body Ignored","identifiesControllerService":false,"sensitive":false,"dynamic":false}},"style":{},"schedulingPeriod":"0 sec","schedulingStrategy":"TIMER_DRIVEN","executionNode":"ALL","penaltyDuration":"30 sec","yieldDuration":"1 sec","bulletinLevel":"WARN","runDurationMillis":0,"concurrentlySchedulableTaskCount":1,"autoTerminatedRelationships":["Original"],"scheduledState":"ENABLED","retryCount":10,"retriedRelationships":[],"backoffMechanism":"PENALIZE_FLOWFILE","maxBackoffPeriod":"10 mins","componentType":"PROCESSOR","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"97cca146-a5f1-376c-8c4b-a98ca43df6d0","instanceIdentifier":"90324add-4289-37c4-2641-825df420489a","name":"CogStackPrepareRecordForOcr","comments":"","position":{"x":-1152.0,"y":-1320.0},"type":"CogStackPrepareRecordForOcr","bundle":{"group":"org.apache.nifi","artifact":"python-extensions","version":"0.0.1"},"properties":{"binary_field_name":"binarydocument","operation_mode":"base64","process_flow_file_type":"avro","output_text_field_name":"text","document_id_field_name":"id"},"propertyDescriptors":{"binary_field_name":{"name":"binary_field_name","displayName":"binary_field_name","identifiesControllerService":false,"sensitive":false,"dynamic":false},"operation_mode":{"name":"operation_mode","displayName":"operation_mode","identifiesControllerService":false,"sensitive":false,"dynamic":false},"process_flow_file_type":{"name":"process_flow_file_type","displayName":"process_flow_file_type","identifiesControllerService":false,"sensitive":false,"dynamic":false},"output_text_field_name":{"name":"output_text_field_name","displayName":"output_text_field_name","identifiesControllerService":false,"sensitive":false,"dynamic":false},"document_id_field_name":{"name":"document_id_field_name","displayName":"document_id_field_name","identifiesControllerService":false,"sensitive":false,"dynamic":false}},"style":{},"schedulingPeriod":"0 sec","schedulingStrategy":"TIMER_DRIVEN","executionNode":"ALL","penaltyDuration":"30 sec","yieldDuration":"1 sec","bulletinLevel":"WARN","runDurationMillis":25,"concurrentlySchedulableTaskCount":1,"autoTerminatedRelationships":["original"],"scheduledState":"ENABLED","retryCount":10,"retriedRelationships":[],"backoffMechanism":"PENALIZE_FLOWFILE","maxBackoffPeriod":"10 mins","componentType":"PROCESSOR","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"fb1931a4-27ef-3ac3-ab5b-120c720f67e7","instanceIdentifier":"a51ace10-fe83-3670-8178-8e009c932a4e","name":"PutElasticsearchJson","comments":"","position":{"x":-496.0,"y":-912.0},"type":"org.apache.nifi.processors.elasticsearch.PutElasticsearchJson","bundle":{"group":"org.apache.nifi","artifact":"nifi-elasticsearch-restapi-nar","version":"2.10.0"},"properties":{"Dynamic Templates":null,"Treat Not Found as Success":"true","Script":null,"Max FlowFiles Per Batch":"100","Max Batch Size":"","Max JSON Field String Length":"20 MB","Index Field":null,"Output Bulk Request":"false","Index Operation":"index","Index":"${generatetablefetch.tableName}","Input Content Format":"Single JSON","Retain Identifier Field":"true","Log Error Responses":"false","Output Error Responses":"false","Identifier Attribute":"id","Retain Index Field":"true","Type":null,"Suppress Nulls":"never-suppress","Timestamp Field":null,"Scripted Upsert":"false","Identifier Field":null,"Client Service":"06280792-dc54-3d7a-9d35-76ce834dcec3","Character Set":"UTF-8","Retain Timestamp Field":"true"},"propertyDescriptors":{"Dynamic Templates":{"name":"Dynamic Templates","displayName":"Dynamic Templates","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Treat Not Found as Success":{"name":"Treat Not Found as Success","displayName":"Treat Not Found as Success","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Script":{"name":"Script","displayName":"Script","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Max FlowFiles Per Batch":{"name":"Max FlowFiles Per Batch","displayName":"Max FlowFiles Per Batch","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Max Batch Size":{"name":"Max Batch Size","displayName":"Max Batch Size","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Max JSON Field String Length":{"name":"Max JSON Field String Length","displayName":"Max JSON Field String Length","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Index Field":{"name":"Index Field","displayName":"Index Field","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Output Bulk Request":{"name":"Output Bulk Request","displayName":"Output Bulk Request","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Index Operation":{"name":"Index Operation","displayName":"Index Operation","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Index":{"name":"Index","displayName":"Index","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Input Content Format":{"name":"Input Content Format","displayName":"Input Content Format","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Retain Identifier Field":{"name":"Retain Identifier Field","displayName":"Retain Identifier Field","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Log Error Responses":{"name":"Log Error Responses","displayName":"Log Error Responses","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Output Error Responses":{"name":"Output Error Responses","displayName":"Output Error Responses","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Identifier Attribute":{"name":"Identifier Attribute","displayName":"Identifier Attribute","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Retain Index Field":{"name":"Retain Index Field","displayName":"Retain Index Field","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Type":{"name":"Type","displayName":"Type","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Suppress Nulls":{"name":"Suppress Nulls","displayName":"Suppress Nulls","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Timestamp Field":{"name":"Timestamp Field","displayName":"Timestamp Field","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Scripted Upsert":{"name":"Scripted Upsert","displayName":"Scripted Upsert","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Identifier Field":{"name":"Identifier Field","displayName":"Identifier Field","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Client Service":{"name":"Client Service","displayName":"Client Service","identifiesControllerService":true,"sensitive":false,"dynamic":false},"Character Set":{"name":"Character Set","displayName":"Character Set","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Retain Timestamp Field":{"name":"Retain Timestamp Field","displayName":"Retain Timestamp Field","identifiesControllerService":false,"sensitive":false,"dynamic":false}},"style":{},"schedulingPeriod":"0 sec","schedulingStrategy":"TIMER_DRIVEN","executionNode":"ALL","penaltyDuration":"30 sec","yieldDuration":"1 sec","bulletinLevel":"WARN","runDurationMillis":0,"concurrentlySchedulableTaskCount":1,"autoTerminatedRelationships":["original","successful"],"scheduledState":"ENABLED","retryCount":10,"retriedRelationships":[],"backoffMechanism":"PENALIZE_FLOWFILE","maxBackoffPeriod":"10 mins","componentType":"PROCESSOR","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"}],"inputPorts":[],"outputPorts":[],"connections":[{"identifier":"fbabb01c-b959-3437-8bc8-554f3709de45","instanceIdentifier":"836e3567-f8c2-3d1a-ba04-bd6f915710a3","name":"","source":{"id":"63d97939-57b6-36a3-91e2-ee0f4cee2b02","type":"PROCESSOR","groupId":"af2b9237-8c32-355c-9495-3be0e06b4c25","name":"SplitJson","comments":"","instanceIdentifier":"b37a4cb7-8b0d-3a5b-4333-df1160bc8967"},"destination":{"id":"97249984-e607-3cf4-a16c-6dc6f5685b17","type":"FUNNEL","groupId":"af2b9237-8c32-355c-9495-3be0e06b4c25","name":"Funnel","comments":"","instanceIdentifier":"1c8a080c-86a8-3b34-8a9b-2ac2ac3e26d8"},"labelIndex":0,"zIndex":29,"selectedRelationships":["failure"],"backPressureObjectThreshold":10000,"backPressureDataSizeThreshold":"1 GB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","partitioningAttribute":"","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"1bf2e59a-3c46-33c6-809c-d92586c396c8","instanceIdentifier":"ed5619c2-a1e9-3241-8277-773484320537","name":"","source":{"id":"b0a7e1fe-0197-1000-7d26-f460ed2eec5e","type":"PROCESSOR","groupId":"af2b9237-8c32-355c-9495-3be0e06b4c25","name":"GenerateTableFetch-encounters","comments":"","instanceIdentifier":"6ddb9b95-fb84-34e6-c230-f4047af5dee9"},"destination":{"id":"d0e96593-ae93-366d-bda1-17ad8ef2ccd0","type":"FUNNEL","groupId":"af2b9237-8c32-355c-9495-3be0e06b4c25","name":"Funnel","comments":"","instanceIdentifier":"74abb30b-c135-3169-c0d0-61d28a9f8970"},"labelIndex":0,"zIndex":2,"selectedRelationships":["failure"],"backPressureObjectThreshold":10000,"backPressureDataSizeThreshold":"1 GB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","partitioningAttribute":"","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"51f84da6-92b7-3971-9cb4-de65ca7359a8","instanceIdentifier":"7d744319-c9fe-391f-9546-7800d4c4c82a","name":"","source":{"id":"63d97939-57b6-36a3-91e2-ee0f4cee2b02","type":"PROCESSOR","groupId":"af2b9237-8c32-355c-9495-3be0e06b4c25","name":"SplitJson","comments":"","instanceIdentifier":"b37a4cb7-8b0d-3a5b-4333-df1160bc8967"},"destination":{"id":"c1de59a6-f215-3d9b-8597-237a1a50852d","type":"PROCESSOR","groupId":"af2b9237-8c32-355c-9495-3be0e06b4c25","name":"EvaluateJsonPath","comments":"","instanceIdentifier":"4c3e5f3c-eb76-3f7f-3714-792ddd0cfb3a"},"labelIndex":0,"zIndex":28,"selectedRelationships":["split"],"backPressureObjectThreshold":10000,"backPressureDataSizeThreshold":"1 GB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","partitioningAttribute":"","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"84055802-dd6e-3ffe-9aa3-91c092fd4f07","instanceIdentifier":"be2c3c8a-c877-327d-623a-38791b9fe20a","name":"","source":{"id":"fb1931a4-27ef-3ac3-ab5b-120c720f67e7","type":"PROCESSOR","groupId":"af2b9237-8c32-355c-9495-3be0e06b4c25","name":"PutElasticsearchJson","comments":"","instanceIdentifier":"a51ace10-fe83-3670-8178-8e009c932a4e"},"destination":{"id":"5ab77308-8a4c-3b4d-9e2f-681858ce03c4","type":"FUNNEL","groupId":"af2b9237-8c32-355c-9495-3be0e06b4c25","name":"Funnel","comments":"","instanceIdentifier":"c08eb90e-6c7d-3470-ac06-b982c66b6acc"},"labelIndex":0,"zIndex":26,"selectedRelationships":["retry"],"backPressureObjectThreshold":10000,"backPressureDataSizeThreshold":"1 GB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","partitioningAttribute":"","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"410f75b5-207d-3ee1-accc-b5bc768a9b20","instanceIdentifier":"7af46b9a-cf54-3bea-4251-df46bbe1118e","name":"","source":{"id":"c1de59a6-f215-3d9b-8597-237a1a50852d","type":"PROCESSOR","groupId":"af2b9237-8c32-355c-9495-3be0e06b4c25","name":"EvaluateJsonPath","comments":"","instanceIdentifier":"4c3e5f3c-eb76-3f7f-3714-792ddd0cfb3a"},"destination":{"id":"8aa474d8-4037-3e6b-81d7-2f50dbb9b2ee","type":"FUNNEL","groupId":"af2b9237-8c32-355c-9495-3be0e06b4c25","name":"Funnel","comments":"","instanceIdentifier":"711fd6bd-ad04-3a40-71d2-f23913a40548"},"labelIndex":0,"zIndex":33,"selectedRelationships":["unmatched"],"backPressureObjectThreshold":10000,"backPressureDataSizeThreshold":"1 GB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","partitioningAttribute":"","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"ff1c2817-832f-3e6b-a748-ecdf9a3735c6","instanceIdentifier":"74d77ab4-019f-1000-d850-e5d69e720451","name":"","source":{"id":"25d6775d-c9fa-3166-9203-973e735e8877","type":"PROCESSOR","groupId":"af2b9237-8c32-355c-9495-3be0e06b4c25","name":"CogStackParseCogStackServiceResult","comments":"","instanceIdentifier":"74d72c6e-019f-1000-bb5f-e95c500a61f5"},"destination":{"id":"1c96b19b-10b0-3f1f-ab60-be8c2b481516","type":"FUNNEL","groupId":"af2b9237-8c32-355c-9495-3be0e06b4c25","name":"Funnel","comments":"","instanceIdentifier":"6068ef4e-8fda-3edd-0145-90e101066568"},"labelIndex":0,"zIndex":40,"selectedRelationships":["failure"],"backPressureObjectThreshold":10000,"backPressureDataSizeThreshold":"1 GB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","partitioningAttribute":"","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"87d954a3-d65b-347c-b2f4-3fe229c206c9","instanceIdentifier":"cbac13fb-17af-30bd-6882-0b7327e0c83e","name":"","source":{"id":"22e4a99b-9b48-3dea-920e-e1600becc662","type":"PROCESSOR","groupId":"af2b9237-8c32-355c-9495-3be0e06b4c25","name":"InvokeHTTP","comments":"","instanceIdentifier":"fb359fe1-f5a9-39b5-94e9-c8dec6112a99"},"destination":{"id":"25d6775d-c9fa-3166-9203-973e735e8877","type":"PROCESSOR","groupId":"af2b9237-8c32-355c-9495-3be0e06b4c25","name":"CogStackParseCogStackServiceResult","comments":"","instanceIdentifier":"74d72c6e-019f-1000-bb5f-e95c500a61f5"},"labelIndex":0,"zIndex":39,"selectedRelationships":["Response"],"backPressureObjectThreshold":10000,"backPressureDataSizeThreshold":"1 GB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","partitioningAttribute":"","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"7156d11f-9e2f-3bfd-b335-e0aa700950ce","instanceIdentifier":"74d79c2c-019f-1000-bf89-93525c704489","name":"","source":{"id":"25d6775d-c9fa-3166-9203-973e735e8877","type":"PROCESSOR","groupId":"af2b9237-8c32-355c-9495-3be0e06b4c25","name":"CogStackParseCogStackServiceResult","comments":"","instanceIdentifier":"74d72c6e-019f-1000-bb5f-e95c500a61f5"},"destination":{"id":"63d97939-57b6-36a3-91e2-ee0f4cee2b02","type":"PROCESSOR","groupId":"af2b9237-8c32-355c-9495-3be0e06b4c25","name":"SplitJson","comments":"","instanceIdentifier":"b37a4cb7-8b0d-3a5b-4333-df1160bc8967"},"labelIndex":0,"zIndex":41,"selectedRelationships":["success"],"backPressureObjectThreshold":10000,"backPressureDataSizeThreshold":"1 GB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","partitioningAttribute":"","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"94f08f0a-202d-35b7-bee5-5387eb15e2d1","instanceIdentifier":"15c35067-101e-3c43-d101-4a530ff30295","name":"","source":{"id":"b0a7e1fe-0197-1000-7d26-f460ed2eec5e","type":"PROCESSOR","groupId":"af2b9237-8c32-355c-9495-3be0e06b4c25","name":"GenerateTableFetch-encounters","comments":"","instanceIdentifier":"6ddb9b95-fb84-34e6-c230-f4047af5dee9"},"destination":{"id":"b0a7e1fd-0197-1000-7799-44654203e24f","type":"PROCESSOR","groupId":"af2b9237-8c32-355c-9495-3be0e06b4c25","name":"ExecuteSQLRecord","comments":"","instanceIdentifier":"c652a637-5daa-3016-69dc-eb043ae56375"},"labelIndex":0,"zIndex":1,"selectedRelationships":["success"],"backPressureObjectThreshold":10000,"backPressureDataSizeThreshold":"1 GB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","partitioningAttribute":"","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"32dbe6de-d258-37d2-86fe-f6b0626c3384","instanceIdentifier":"1e0a1010-b5ab-3420-ff1a-12383d69982c","name":"","source":{"id":"fb1931a4-27ef-3ac3-ab5b-120c720f67e7","type":"PROCESSOR","groupId":"af2b9237-8c32-355c-9495-3be0e06b4c25","name":"PutElasticsearchJson","comments":"","instanceIdentifier":"a51ace10-fe83-3670-8178-8e009c932a4e"},"destination":{"id":"386800ca-c242-37d0-981a-5d12a5cf0d6e","type":"FUNNEL","groupId":"af2b9237-8c32-355c-9495-3be0e06b4c25","name":"Funnel","comments":"","instanceIdentifier":"413d2c24-9b99-3911-ed11-8ce656a5d68f"},"labelIndex":0,"zIndex":27,"selectedRelationships":["failure","errors"],"backPressureObjectThreshold":10000,"backPressureDataSizeThreshold":"1 GB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","partitioningAttribute":"","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"d525d30e-7744-323c-a6c6-e557aacfef14","instanceIdentifier":"b623aea0-830a-3ccd-db51-9dd74f3cc00b","name":"","source":{"id":"b0a7e1fd-0197-1000-7799-44654203e24f","type":"PROCESSOR","groupId":"af2b9237-8c32-355c-9495-3be0e06b4c25","name":"ExecuteSQLRecord","comments":"","instanceIdentifier":"c652a637-5daa-3016-69dc-eb043ae56375"},"destination":{"id":"97cca146-a5f1-376c-8c4b-a98ca43df6d0","type":"PROCESSOR","groupId":"af2b9237-8c32-355c-9495-3be0e06b4c25","name":"CogStackPrepareRecordForOcr","comments":"","instanceIdentifier":"90324add-4289-37c4-2641-825df420489a"},"labelIndex":0,"zIndex":35,"selectedRelationships":["success"],"backPressureObjectThreshold":10000,"backPressureDataSizeThreshold":"1 GB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","partitioningAttribute":"","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"89389b4e-fe64-360d-94c7-d0ea68444b3c","instanceIdentifier":"c3818a5a-b2ea-3b22-aa1b-09d99a258fa5","name":"","source":{"id":"b0a7e1fd-0197-1000-7799-44654203e24f","type":"PROCESSOR","groupId":"af2b9237-8c32-355c-9495-3be0e06b4c25","name":"ExecuteSQLRecord","comments":"","instanceIdentifier":"c652a637-5daa-3016-69dc-eb043ae56375"},"destination":{"id":"b0dc6924-0197-1000-57e8-8dbd2331558e","type":"FUNNEL","groupId":"af2b9237-8c32-355c-9495-3be0e06b4c25","name":"Funnel","comments":"","instanceIdentifier":"ea2890f4-0312-3463-45fe-9c0495ee0edb"},"labelIndex":0,"zIndex":4,"selectedRelationships":["failure"],"backPressureObjectThreshold":10000,"backPressureDataSizeThreshold":"1 GB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","partitioningAttribute":"","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"6f12409a-5ed1-36d9-b9c4-c82f81a7b571","instanceIdentifier":"1bf02a09-d044-3911-2a2c-8b6b8376bf19","name":"","source":{"id":"c1de59a6-f215-3d9b-8597-237a1a50852d","type":"PROCESSOR","groupId":"af2b9237-8c32-355c-9495-3be0e06b4c25","name":"EvaluateJsonPath","comments":"","instanceIdentifier":"4c3e5f3c-eb76-3f7f-3714-792ddd0cfb3a"},"destination":{"id":"32097907-c0d5-3475-89d9-8801f540a380","type":"FUNNEL","groupId":"af2b9237-8c32-355c-9495-3be0e06b4c25","name":"Funnel","comments":"","instanceIdentifier":"20ece14b-f247-3652-6b4f-93a15b914d8f"},"labelIndex":0,"zIndex":32,"selectedRelationships":["failure"],"backPressureObjectThreshold":10000,"backPressureDataSizeThreshold":"1 GB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","partitioningAttribute":"","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"9e7dfd87-7442-3d32-8cb1-2bdc9fec2838","instanceIdentifier":"49d6a0fe-7c7e-39cf-7207-13e881efab9b","name":"","source":{"id":"22e4a99b-9b48-3dea-920e-e1600becc662","type":"PROCESSOR","groupId":"af2b9237-8c32-355c-9495-3be0e06b4c25","name":"InvokeHTTP","comments":"","instanceIdentifier":"fb359fe1-f5a9-39b5-94e9-c8dec6112a99"},"destination":{"id":"b7c65dca-19eb-3c74-bdfa-a436a086e0cd","type":"FUNNEL","groupId":"af2b9237-8c32-355c-9495-3be0e06b4c25","name":"Funnel","comments":"","instanceIdentifier":"595372fd-5b1d-35b2-b695-d9fde917f226"},"labelIndex":0,"zIndex":8,"selectedRelationships":["No Retry","Retry","Failure"],"backPressureObjectThreshold":10000,"backPressureDataSizeThreshold":"1 GB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","partitioningAttribute":"","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"a7e28e19-fe7a-3362-ab67-0923fd662218","instanceIdentifier":"1bd73264-5323-3125-fc4d-d3b8598f17ba","name":"","source":{"id":"c1de59a6-f215-3d9b-8597-237a1a50852d","type":"PROCESSOR","groupId":"af2b9237-8c32-355c-9495-3be0e06b4c25","name":"EvaluateJsonPath","comments":"","instanceIdentifier":"4c3e5f3c-eb76-3f7f-3714-792ddd0cfb3a"},"destination":{"id":"fb1931a4-27ef-3ac3-ab5b-120c720f67e7","type":"PROCESSOR","groupId":"af2b9237-8c32-355c-9495-3be0e06b4c25","name":"PutElasticsearchJson","comments":"","instanceIdentifier":"a51ace10-fe83-3670-8178-8e009c932a4e"},"labelIndex":0,"zIndex":31,"selectedRelationships":["matched"],"backPressureObjectThreshold":10000,"backPressureDataSizeThreshold":"1 GB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","partitioningAttribute":"","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"97bcd005-0144-370c-9c92-1919663d1032","instanceIdentifier":"932f4afa-687b-37ce-1e2c-e57d638ea841","name":"","source":{"id":"97cca146-a5f1-376c-8c4b-a98ca43df6d0","type":"PROCESSOR","groupId":"af2b9237-8c32-355c-9495-3be0e06b4c25","name":"CogStackPrepareRecordForOcr","comments":"","instanceIdentifier":"90324add-4289-37c4-2641-825df420489a"},"destination":{"id":"22e4a99b-9b48-3dea-920e-e1600becc662","type":"PROCESSOR","groupId":"af2b9237-8c32-355c-9495-3be0e06b4c25","name":"InvokeHTTP","comments":"","instanceIdentifier":"fb359fe1-f5a9-39b5-94e9-c8dec6112a99"},"labelIndex":0,"zIndex":36,"selectedRelationships":["success"],"backPressureObjectThreshold":10000,"backPressureDataSizeThreshold":"1 GB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","partitioningAttribute":"","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"d92ca9c0-f829-307e-83a5-0ad2d8845374","instanceIdentifier":"a3495c8f-cd66-3c38-2921-a98f6dfefe33","name":"","source":{"id":"97cca146-a5f1-376c-8c4b-a98ca43df6d0","type":"PROCESSOR","groupId":"af2b9237-8c32-355c-9495-3be0e06b4c25","name":"CogStackPrepareRecordForOcr","comments":"","instanceIdentifier":"90324add-4289-37c4-2641-825df420489a"},"destination":{"id":"3898e396-9a16-387d-bffa-b1ae83c622e0","type":"FUNNEL","groupId":"af2b9237-8c32-355c-9495-3be0e06b4c25","name":"Funnel","comments":"","instanceIdentifier":"83f0bc54-1645-3ee8-77d9-172120575bc8"},"labelIndex":0,"zIndex":34,"selectedRelationships":["failure"],"backPressureObjectThreshold":10000,"backPressureDataSizeThreshold":"1 GB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","partitioningAttribute":"","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"}],"labels":[{"identifier":"0ccecdf2-914a-32fb-bf20-ea67ea924b9a","instanceIdentifier":"a1e137a9-3e80-3f54-aa4c-2f97dbbcf03f","position":{"x":-792.0,"y":-1120.0},"label":"OCR-ing takes place here, please make sure you have ocr-service/ocr-service-text-only Docker container running, and the ports 8090 (for ocr-ing) / 8091 (text extraction only) are free. Note: if both services are running on the same host as NiFi use port 8090 for both urls: http://ocr-service:8090/api/process or http://ocr-service-text-only:8090/api/process","zIndex":2,"width":296.0,"height":160.0,"style":{"font-size":"12px"},"componentType":"LABEL","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"760c2491-0198-1000-3cfc-b0cebed70256","instanceIdentifier":"7b399dfa-c586-397a-ac9c-21b70cc3e5cd","position":{"x":-1152.0,"y":-544.0},"label":"We split the json because we want 1 record per flowfile, so that we can later update the attributes of the flowfile withe the record's UUID.\n\nIMPORTANT: note the path expression : $.*","zIndex":2,"width":352.0,"height":104.0,"style":{"font-size":"12px"},"componentType":"LABEL","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"30b8928d-679e-3c15-b5b4-ce7efcde9231","instanceIdentifier":"f55a587b-9629-3531-6533-ab166e1aa2e7","position":{"x":-152.0,"y":-1072.0},"label":"Notice that the ElasticsearchJson processor supports only insertion by attribute name, as opposed to the ElasticsearchRecord processor that takes the desired identifier directly from the avro flowfile.","zIndex":4,"width":288.0,"height":136.0,"style":{"font-size":"12px"},"componentType":"LABEL","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"76037ca7-0198-1000-e6d6-7366ac6b8d44","instanceIdentifier":"2e2039b6-e98f-3149-b037-3b3b9908af1a","position":{"x":-1160.0,"y":-1760.0},"label":"Simple ingestion from PgSQL with OCR to ElasticSearch","zIndex":1,"width":368.0,"height":24.0,"style":{"font-size":"12px"},"componentType":"LABEL","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"d1458db0-dccc-3554-98b1-2e54c2937481","instanceIdentifier":"5df7243f-05a2-3500-0966-dc19a63e80a3","position":{"x":-136.0,"y":-584.0},"label":"Check the EvaluateJsonPath, we add a new property called \"id\" with the value \"$.id\" , takes the value from the json.\n","zIndex":3,"width":320.0,"height":104.0,"style":{"font-size":"12px"},"componentType":"LABEL","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"}],"funnels":[{"identifier":"32097907-c0d5-3475-89d9-8801f540a380","instanceIdentifier":"20ece14b-f247-3652-6b4f-93a15b914d8f","position":{"x":-344.0,"y":-496.0},"componentType":"FUNNEL","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"b0dc6924-0197-1000-57e8-8dbd2331558e","instanceIdentifier":"ea2890f4-0312-3463-45fe-9c0495ee0edb","position":{"x":-1480.0,"y":-1488.0},"componentType":"FUNNEL","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"d0e96593-ae93-366d-bda1-17ad8ef2ccd0","instanceIdentifier":"74abb30b-c135-3169-c0d0-61d28a9f8970","position":{"x":-1480.0,"y":-1680.0},"componentType":"FUNNEL","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"97249984-e607-3cf4-a16c-6dc6f5685b17","instanceIdentifier":"1c8a080c-86a8-3b34-8a9b-2ac2ac3e26d8","position":{"x":-1480.0,"y":-640.0},"componentType":"FUNNEL","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"3898e396-9a16-387d-bffa-b1ae83c622e0","instanceIdentifier":"83f0bc54-1645-3ee8-77d9-172120575bc8","position":{"x":-1480.0,"y":-1280.0},"componentType":"FUNNEL","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"8aa474d8-4037-3e6b-81d7-2f50dbb9b2ee","instanceIdentifier":"711fd6bd-ad04-3a40-71d2-f23913a40548","position":{"x":136.0,"y":-656.0},"componentType":"FUNNEL","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"386800ca-c242-37d0-981a-5d12a5cf0d6e","instanceIdentifier":"413d2c24-9b99-3911-ed11-8ce656a5d68f","position":{"x":-344.0,"y":-1032.0},"componentType":"FUNNEL","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"b7c65dca-19eb-3c74-bdfa-a436a086e0cd","instanceIdentifier":"595372fd-5b1d-35b2-b695-d9fde917f226","position":{"x":-1480.0,"y":-1080.0},"componentType":"FUNNEL","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"1c96b19b-10b0-3f1f-ab60-be8c2b481516","instanceIdentifier":"6068ef4e-8fda-3edd-0145-90e101066568","position":{"x":-1480.0,"y":-856.0},"componentType":"FUNNEL","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"5ab77308-8a4c-3b4d-9e2f-681858ce03c4","instanceIdentifier":"c08eb90e-6c7d-3470-ac06-b982c66b6acc","position":{"x":136.0,"y":-864.0},"componentType":"FUNNEL","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"}],"controllerServices":[{"identifier":"cf502be1-705e-3731-95b7-2dbd5c603c26","instanceIdentifier":"4b0cd571-6a65-3785-18cd-7f92468e85f4","name":"StandardSSLContextService","comments":"","type":"org.apache.nifi.ssl.StandardSSLContextService","bundle":{"group":"org.apache.nifi","artifact":"nifi-ssl-context-service-nar","version":"2.10.0"},"properties":{"Truststore Type":"JKS","Keystore Type":"JKS","Truststore Filename":"/security/certificates/elastic/opensearch/elasticsearch/elasticsearch-1/elasticsearch-1-truststore.key","TLS Protocol":"TLS","Keystore Filename":"/security/certificates/elastic/opensearch/elasticsearch/elasticsearch-1/elasticsearch-1-keystore.jks"},"propertyDescriptors":{"Truststore Type":{"name":"Truststore Type","displayName":"Truststore Type","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Key Password":{"name":"Key Password","displayName":"Key Password","identifiesControllerService":false,"sensitive":true,"dynamic":false},"Keystore Type":{"name":"Keystore Type","displayName":"Keystore Type","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Truststore Filename":{"name":"Truststore Filename","displayName":"Truststore Filename","identifiesControllerService":false,"sensitive":false,"dynamic":false,"resourceDefinition":{"cardinality":"SINGLE","resourceTypes":["FILE"]}},"Keystore Password":{"name":"Keystore Password","displayName":"Keystore Password","identifiesControllerService":false,"sensitive":true,"dynamic":false},"Truststore Password":{"name":"Truststore Password","displayName":"Truststore Password","identifiesControllerService":false,"sensitive":true,"dynamic":false},"TLS Protocol":{"name":"TLS Protocol","displayName":"TLS Protocol","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Keystore Filename":{"name":"Keystore Filename","displayName":"Keystore Filename","identifiesControllerService":false,"sensitive":false,"dynamic":false,"resourceDefinition":{"cardinality":"SINGLE","resourceTypes":["FILE"]}}},"controllerServiceApis":[{"type":"org.apache.nifi.ssl.SSLContextProvider","bundle":{"group":"org.apache.nifi","artifact":"nifi-standard-services-api-nar","version":"2.10.0"}},{"type":"org.apache.nifi.ssl.SSLContextService","bundle":{"group":"org.apache.nifi","artifact":"nifi-standard-services-api-nar","version":"2.10.0"}}],"scheduledState":"DISABLED","bulletinLevel":"WARN","componentType":"CONTROLLER_SERVICE","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"ae4d900f-d06e-3cc3-acc5-4f6f222a5064","instanceIdentifier":"62a09a84-3bee-3ffd-b5d0-031d1b7e8cee","name":"DBCPConnectionPool","comments":"","type":"org.apache.nifi.dbcp.DBCPConnectionPool","bundle":{"group":"org.apache.nifi","artifact":"nifi-dbcp-service-nar","version":"2.10.0"},"properties":{"Maximum Idle Connections":"8","Database Driver Locations":"/opt/nifi/drivers/postgresql-42.7.7.jar","Max Wait Time":"500 millis","Database Driver Class Name":"org.postgresql.Driver","Minimum Idle Connections":"0","Kerberos User Service":null,"Max Total Connections":"8","Validation Query":null,"Database Password Provider":null,"Database Connection URL":"jdbc:postgresql://samples-db:5432/db_samples","Password Source":"PASSWORD","Time Between Eviction Runs":"-1","Database User":"test","Soft Minimum Evictable Idle Time":"-1","Maximum Connection Lifetime":"-1","Minimum Evictable Idle Time":"30 mins"},"propertyDescriptors":{"Maximum Idle Connections":{"name":"Maximum Idle Connections","displayName":"Maximum Idle Connections","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Database Driver Locations":{"name":"Database Driver Locations","displayName":"Database Driver Locations","identifiesControllerService":false,"sensitive":false,"dynamic":false,"resourceDefinition":{"cardinality":"MULTIPLE","resourceTypes":["FILE","DIRECTORY","URL"]}},"Max Wait Time":{"name":"Max Wait Time","displayName":"Max Wait Time","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Database Driver Class Name":{"name":"Database Driver Class Name","displayName":"Database Driver Class Name","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Minimum Idle Connections":{"name":"Minimum Idle Connections","displayName":"Minimum Idle Connections","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Kerberos User Service":{"name":"Kerberos User Service","displayName":"Kerberos User Service","identifiesControllerService":true,"sensitive":false,"dynamic":false},"Max Total Connections":{"name":"Max Total Connections","displayName":"Max Total Connections","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Validation Query":{"name":"Validation Query","displayName":"Validation Query","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Database Password Provider":{"name":"Database Password Provider","displayName":"Database Password Provider","identifiesControllerService":true,"sensitive":false,"dynamic":false},"Database Connection URL":{"name":"Database Connection URL","displayName":"Database Connection URL","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Password Source":{"name":"Password Source","displayName":"Password Source","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Time Between Eviction Runs":{"name":"Time Between Eviction Runs","displayName":"Time Between Eviction Runs","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Database User":{"name":"Database User","displayName":"Database User","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Soft Minimum Evictable Idle Time":{"name":"Soft Minimum Evictable Idle Time","displayName":"Soft Minimum Evictable Idle Time","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Maximum Connection Lifetime":{"name":"Maximum Connection Lifetime","displayName":"Maximum Connection Lifetime","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Minimum Evictable Idle Time":{"name":"Minimum Evictable Idle Time","displayName":"Minimum Evictable Idle Time","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Password":{"name":"Password","displayName":"Password","identifiesControllerService":false,"sensitive":true,"dynamic":false}},"controllerServiceApis":[{"type":"org.apache.nifi.dbcp.DBCPService","bundle":{"group":"org.apache.nifi","artifact":"nifi-standard-services-api-nar","version":"2.10.0"}}],"scheduledState":"DISABLED","bulletinLevel":"WARN","componentType":"CONTROLLER_SERVICE","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"06280792-dc54-3d7a-9d35-76ce834dcec3","instanceIdentifier":"5418c689-1502-3458-f583-b82f274a01ac","name":"ElasticSearchClientServiceImpl","comments":"","type":"org.apache.nifi.elasticsearch.ElasticSearchClientServiceImpl","bundle":{"group":"org.apache.nifi","artifact":"nifi-elasticsearch-client-service-nar","version":"2.10.0"},"properties":{"Sniffer Request Timeout":"1 sec","Connect Timeout":"5000","HTTP Hosts":"https://elasticsearch-1:9200","Path Prefix":null,"API Key ID":null,"Enable Compression":"false","Read Timeout":"60000","Sniff on Failure":"false","Sniffer Failure Delay":"1 min","OAuth2 Access Token Provider":null,"Suppress Null and Empty Values":"always-suppress","Node Selector":"ANY","Proxy Configuration Service":null,"Strict Deprecation":"false","Sniff Cluster Nodes":"false","SSL Context Service":"cf502be1-705e-3731-95b7-2dbd5c603c26","Send Meta Header":"true","Username":"admin","Authorization Scheme":"BASIC","Sniffer Interval":"5 mins","Run As User":null,"Character Set":"UTF-8"},"propertyDescriptors":{"Sniffer Request Timeout":{"name":"Sniffer Request Timeout","displayName":"Sniffer Request Timeout","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Connect Timeout":{"name":"Connect Timeout","displayName":"Connect Timeout","identifiesControllerService":false,"sensitive":false,"dynamic":false},"API Key ID":{"name":"API Key ID","displayName":"API Key ID","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Enable Compression":{"name":"Enable Compression","displayName":"Enable Compression","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Sniff on Failure":{"name":"Sniff on Failure","displayName":"Sniff on Failure","identifiesControllerService":false,"sensitive":false,"dynamic":false},"OAuth2 Access Token Provider":{"name":"OAuth2 Access Token Provider","displayName":"OAuth2 Access Token Provider","identifiesControllerService":true,"sensitive":false,"dynamic":false},"Node Selector":{"name":"Node Selector","displayName":"Node Selector","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Proxy Configuration Service":{"name":"Proxy Configuration Service","displayName":"Proxy Configuration Service","identifiesControllerService":true,"sensitive":false,"dynamic":false},"Strict Deprecation":{"name":"Strict Deprecation","displayName":"Strict Deprecation","identifiesControllerService":false,"sensitive":false,"dynamic":false},"JWT Shared Secret":{"name":"JWT Shared Secret","displayName":"JWT Shared Secret","identifiesControllerService":false,"sensitive":true,"dynamic":false},"API Key":{"name":"API Key","displayName":"API Key","identifiesControllerService":false,"sensitive":true,"dynamic":false},"Send Meta Header":{"name":"Send Meta Header","displayName":"Send Meta Header","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Authorization Scheme":{"name":"Authorization Scheme","displayName":"Authorization Scheme","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Password":{"name":"Password","displayName":"Password","identifiesControllerService":false,"sensitive":true,"dynamic":false},"HTTP Hosts":{"name":"HTTP Hosts","displayName":"HTTP Hosts","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Path Prefix":{"name":"Path Prefix","displayName":"Path Prefix","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Read Timeout":{"name":"Read Timeout","displayName":"Read Timeout","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Sniffer Failure Delay":{"name":"Sniffer Failure Delay","displayName":"Sniffer Failure Delay","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Suppress Null and Empty Values":{"name":"Suppress Null and Empty Values","displayName":"Suppress Null and Empty Values","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Sniff Cluster Nodes":{"name":"Sniff Cluster Nodes","displayName":"Sniff Cluster Nodes","identifiesControllerService":false,"sensitive":false,"dynamic":false},"SSL Context Service":{"name":"SSL Context Service","displayName":"SSL Context Service","identifiesControllerService":true,"sensitive":false,"dynamic":false},"Username":{"name":"Username","displayName":"Username","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Sniffer Interval":{"name":"Sniffer Interval","displayName":"Sniffer Interval","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Run As User":{"name":"Run As User","displayName":"Run As User","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Character Set":{"name":"Character Set","displayName":"Character Set","identifiesControllerService":false,"sensitive":false,"dynamic":false}},"controllerServiceApis":[{"type":"org.apache.nifi.elasticsearch.ElasticSearchClientService","bundle":{"group":"org.apache.nifi","artifact":"nifi-elasticsearch-client-service-api-nar","version":"2.10.0"}}],"scheduledState":"DISABLED","bulletinLevel":"WARN","componentType":"CONTROLLER_SERVICE","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"4ad0bf71-a5de-39f7-9f74-07213d115a9f","instanceIdentifier":"8411d042-d05f-3bf6-7f0e-2c12cf8ed007","name":"AvroRecordSetWriter","comments":"","type":"org.apache.nifi.avro.AvroRecordSetWriter","bundle":{"group":"org.apache.nifi","artifact":"nifi-record-serialization-services-nar","version":"2.10.0"},"properties":{"Schema Branch":null,"Schema Reference Reader":null,"Schema Text":"${avro.schema}","Schema Registry":null,"Schema Access Strategy":"inherit-record-schema","Schema Name":"${schema.name}","Schema Write Strategy":"avro-embedded","Encoder Pool Size":"32","Compression Format":"NONE","Schema Cache":null,"Schema Version":null,"Cache Size":"1000","Schema Reference Writer":null},"propertyDescriptors":{"Schema Branch":{"name":"Schema Branch","displayName":"Schema Branch","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Schema Reference Reader":{"name":"Schema Reference Reader","displayName":"Schema Reference Reader","identifiesControllerService":true,"sensitive":false,"dynamic":false},"Schema Text":{"name":"Schema Text","displayName":"Schema Text","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Schema Registry":{"name":"Schema Registry","displayName":"Schema Registry","identifiesControllerService":true,"sensitive":false,"dynamic":false},"Schema Access Strategy":{"name":"Schema Access Strategy","displayName":"Schema Access Strategy","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Schema Name":{"name":"Schema Name","displayName":"Schema Name","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Schema Write Strategy":{"name":"Schema Write Strategy","displayName":"Schema Write Strategy","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Encoder Pool Size":{"name":"Encoder Pool Size","displayName":"Encoder Pool Size","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Compression Format":{"name":"Compression Format","displayName":"Compression Format","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Schema Cache":{"name":"Schema Cache","displayName":"Schema Cache","identifiesControllerService":true,"sensitive":false,"dynamic":false},"Schema Version":{"name":"Schema Version","displayName":"Schema Version","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Cache Size":{"name":"Cache Size","displayName":"Cache Size","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Schema Reference Writer":{"name":"Schema Reference Writer","displayName":"Schema Reference Writer","identifiesControllerService":true,"sensitive":false,"dynamic":false}},"controllerServiceApis":[{"type":"org.apache.nifi.serialization.RecordSetWriterFactory","bundle":{"group":"org.apache.nifi","artifact":"nifi-standard-services-api-nar","version":"2.10.0"}}],"scheduledState":"DISABLED","bulletinLevel":"WARN","componentType":"CONTROLLER_SERVICE","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"}],"defaultFlowFileExpiration":"0 sec","defaultBackPressureObjectThreshold":10000,"defaultBackPressureDataSizeThreshold":"1 GB","scheduledState":"ENABLED","executionEngine":"INHERITED","maxConcurrentTasks":1,"statelessFlowTimeout":"1 min","flowFileConcurrency":"UNBOUNDED","flowFileOutboundPolicy":"STREAM_WHEN_AVAILABLE","componentType":"PROCESS_GROUP"},"externalControllerServices":{},"parameterContexts":{},"flowEncodingVersion":"1.0","parameterProviders":{},"latest":false} \ No newline at end of file +{"flowContents":{"identifier":"af2b9237-8c32-355c-9495-3be0e06b4c25","instanceIdentifier":"74590095-019f-1000-9c76-52dada17636f","name":"opensearch_ingest_docs_db_ocr_service_to_es","comments":"","position":{"x":872.0,"y":352.0},"processGroups":[],"remoteProcessGroups":[],"processors":[{"identifier":"b0a7e1fd-0197-1000-7799-44654203e24f","instanceIdentifier":"c652a637-5daa-3016-69dc-eb043ae56375","name":"ExecuteSQLRecord","comments":"","position":{"x":-1152.0,"y":-1528.0},"type":"org.apache.nifi.processors.standard.ExecuteSQLRecord","bundle":{"group":"org.apache.nifi","artifact":"nifi-standard-nar","version":"2.10.0"},"properties":{"Set Auto Commit":"true","Max Wait Time":"0 seconds","SQL Query":null,"Normalize Table/Column Names":"false","Database Connection Pooling Service":"ae4d900f-d06e-3cc3-acc5-4f6f222a5064","Default Decimal Scale":"0","SQL Post-Query":null,"Fetch Size":"0","Output Batch Size":"0","Use Avro Logical Types":"true","Record Writer":"4ad0bf71-a5de-39f7-9f74-07213d115a9f","SQL Pre-Query":null,"Default Decimal Precision":"10","Max Rows Per FlowFile":"1"},"propertyDescriptors":{"Set Auto Commit":{"name":"Set Auto Commit","displayName":"Set Auto Commit","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Max Wait Time":{"name":"Max Wait Time","displayName":"Max Wait Time","identifiesControllerService":false,"sensitive":false,"dynamic":false},"SQL Query":{"name":"SQL Query","displayName":"SQL Query","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Normalize Table/Column Names":{"name":"Normalize Table/Column Names","displayName":"Normalize Table/Column Names","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Database Connection Pooling Service":{"name":"Database Connection Pooling Service","displayName":"Database Connection Pooling Service","identifiesControllerService":true,"sensitive":false,"dynamic":false},"Default Decimal Scale":{"name":"Default Decimal Scale","displayName":"Default Decimal Scale","identifiesControllerService":false,"sensitive":false,"dynamic":false},"SQL Post-Query":{"name":"SQL Post-Query","displayName":"SQL Post-Query","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Fetch Size":{"name":"Fetch Size","displayName":"Fetch Size","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Output Batch Size":{"name":"Output Batch Size","displayName":"Output Batch Size","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Use Avro Logical Types":{"name":"Use Avro Logical Types","displayName":"Use Avro Logical Types","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Record Writer":{"name":"Record Writer","displayName":"Record Writer","identifiesControllerService":true,"sensitive":false,"dynamic":false},"SQL Pre-Query":{"name":"SQL Pre-Query","displayName":"SQL Pre-Query","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Default Decimal Precision":{"name":"Default Decimal Precision","displayName":"Default Decimal Precision","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Max Rows Per FlowFile":{"name":"Max Rows Per FlowFile","displayName":"Max Rows Per FlowFile","identifiesControllerService":false,"sensitive":false,"dynamic":false}},"style":{},"schedulingPeriod":"0 sec","schedulingStrategy":"TIMER_DRIVEN","executionNode":"ALL","penaltyDuration":"30 sec","yieldDuration":"1 sec","bulletinLevel":"WARN","runDurationMillis":0,"concurrentlySchedulableTaskCount":1,"autoTerminatedRelationships":[],"scheduledState":"ENABLED","retryCount":10,"retriedRelationships":[],"backoffMechanism":"PENALIZE_FLOWFILE","maxBackoffPeriod":"10 mins","componentType":"PROCESSOR","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"63d97939-57b6-36a3-91e2-ee0f4cee2b02","instanceIdentifier":"b37a4cb7-8b0d-3a5b-4333-df1160bc8967","name":"SplitJson","comments":"","position":{"x":-1152.0,"y":-688.0},"type":"org.apache.nifi.processors.standard.SplitJson","bundle":{"group":"org.apache.nifi","artifact":"nifi-standard-nar","version":"2.10.0"},"properties":{"Max String Length":"20 MB","Null Value Representation":"empty string","JsonPath Expression":"$.*"},"propertyDescriptors":{"Max String Length":{"name":"Max String Length","displayName":"Max String Length","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Null Value Representation":{"name":"Null Value Representation","displayName":"Null Value Representation","identifiesControllerService":false,"sensitive":false,"dynamic":false},"JsonPath Expression":{"name":"JsonPath Expression","displayName":"JsonPath Expression","identifiesControllerService":false,"sensitive":false,"dynamic":false}},"style":{},"schedulingPeriod":"0 sec","schedulingStrategy":"TIMER_DRIVEN","executionNode":"ALL","penaltyDuration":"30 sec","yieldDuration":"1 sec","bulletinLevel":"WARN","runDurationMillis":0,"concurrentlySchedulableTaskCount":1,"autoTerminatedRelationships":["original"],"scheduledState":"ENABLED","retryCount":10,"retriedRelationships":[],"backoffMechanism":"PENALIZE_FLOWFILE","maxBackoffPeriod":"10 mins","componentType":"PROCESSOR","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"b0a7e1fe-0197-1000-7d26-f460ed2eec5e","instanceIdentifier":"6ddb9b95-fb84-34e6-c230-f4047af5dee9","name":"GenerateTableFetch-encounters","comments":"","position":{"x":-1152.0,"y":-1720.0},"type":"org.apache.nifi.processors.standard.GenerateTableFetch","bundle":{"group":"org.apache.nifi","artifact":"nifi-standard-nar","version":"2.10.0"},"properties":{"Table Name":"encounters","Max Wait Time":"0 seconds","Partition Size":"1000","Database Connection Pooling Service":"ae4d900f-d06e-3cc3-acc5-4f6f222a5064","Database Type":"PostgreSQL","Columns to Return":null,"Maximum-value Columns":"cid","Custom ORDER BY Column":null,"Output Empty FlowFile on Zero Results":"false","Database Dialect Service":null,"Column for Value Partitioning":"cid","Additional WHERE Clause":null},"propertyDescriptors":{"Table Name":{"name":"Table Name","displayName":"Table Name","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Max Wait Time":{"name":"Max Wait Time","displayName":"Max Wait Time","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Partition Size":{"name":"Partition Size","displayName":"Partition Size","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Database Connection Pooling Service":{"name":"Database Connection Pooling Service","displayName":"Database Connection Pooling Service","identifiesControllerService":true,"sensitive":false,"dynamic":false},"Database Type":{"name":"Database Type","displayName":"Database Type","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Columns to Return":{"name":"Columns to Return","displayName":"Columns to Return","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Maximum-value Columns":{"name":"Maximum-value Columns","displayName":"Maximum-value Columns","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Custom ORDER BY Column":{"name":"Custom ORDER BY Column","displayName":"Custom ORDER BY Column","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Output Empty FlowFile on Zero Results":{"name":"Output Empty FlowFile on Zero Results","displayName":"Output Empty FlowFile on Zero Results","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Database Dialect Service":{"name":"Database Dialect Service","displayName":"Database Dialect Service","identifiesControllerService":true,"sensitive":false,"dynamic":false},"Column for Value Partitioning":{"name":"Column for Value Partitioning","displayName":"Column for Value Partitioning","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Additional WHERE Clause":{"name":"Additional WHERE Clause","displayName":"Additional WHERE Clause","identifiesControllerService":false,"sensitive":false,"dynamic":false}},"style":{},"schedulingPeriod":"1 sec","schedulingStrategy":"TIMER_DRIVEN","executionNode":"ALL","penaltyDuration":"30 sec","yieldDuration":"1 sec","bulletinLevel":"WARN","runDurationMillis":0,"concurrentlySchedulableTaskCount":1,"autoTerminatedRelationships":[],"scheduledState":"ENABLED","retryCount":10,"retriedRelationships":[],"backoffMechanism":"PENALIZE_FLOWFILE","maxBackoffPeriod":"10 mins","componentType":"PROCESSOR","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"25d6775d-c9fa-3166-9203-973e735e8877","instanceIdentifier":"74d72c6e-019f-1000-bb5f-e95c500a61f5","name":"CogStackParseCogStackServiceResult","comments":"","position":{"x":-1152.0,"y":-896.0},"type":"CogStackParseCogStackServiceResult","bundle":{"group":"org.apache.nifi","artifact":"python-extensions","version":"0.0.3"},"properties":{"medcat_empty_text_handling":"drop","document_text_field_name":"text","medcat_no_annotations_handling":"restore","medcat_output_mode":"not_set","service_message_type":"ocr","output_text_field_name":"text","document_id_field_name":"id","medcat_deid_keep_annotations":"true"},"propertyDescriptors":{"medcat_empty_text_handling":{"name":"medcat_empty_text_handling","displayName":"medcat_empty_text_handling","identifiesControllerService":false,"sensitive":false,"dynamic":false},"document_text_field_name":{"name":"document_text_field_name","displayName":"document_text_field_name","identifiesControllerService":false,"sensitive":false,"dynamic":false},"medcat_no_annotations_handling":{"name":"medcat_no_annotations_handling","displayName":"medcat_no_annotations_handling","identifiesControllerService":false,"sensitive":false,"dynamic":false},"medcat_output_mode":{"name":"medcat_output_mode","displayName":"medcat_output_mode","identifiesControllerService":false,"sensitive":false,"dynamic":false},"service_message_type":{"name":"service_message_type","displayName":"service_message_type","identifiesControllerService":false,"sensitive":false,"dynamic":false},"output_text_field_name":{"name":"output_text_field_name","displayName":"output_text_field_name","identifiesControllerService":false,"sensitive":false,"dynamic":false},"document_id_field_name":{"name":"document_id_field_name","displayName":"document_id_field_name","identifiesControllerService":false,"sensitive":false,"dynamic":false},"medcat_deid_keep_annotations":{"name":"medcat_deid_keep_annotations","displayName":"medcat_deid_keep_annotations","identifiesControllerService":false,"sensitive":false,"dynamic":false}},"style":{},"schedulingPeriod":"0 sec","schedulingStrategy":"TIMER_DRIVEN","executionNode":"ALL","penaltyDuration":"30 sec","yieldDuration":"1 sec","bulletinLevel":"WARN","runDurationMillis":25,"concurrentlySchedulableTaskCount":1,"autoTerminatedRelationships":["original"],"scheduledState":"ENABLED","retryCount":10,"retriedRelationships":[],"backoffMechanism":"PENALIZE_FLOWFILE","maxBackoffPeriod":"10 mins","componentType":"PROCESSOR","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"c1de59a6-f215-3d9b-8597-237a1a50852d","instanceIdentifier":"4c3e5f3c-eb76-3f7f-3714-792ddd0cfb3a","name":"EvaluateJsonPath","comments":"","position":{"x":-496.0,"y":-688.0},"type":"org.apache.nifi.processors.standard.EvaluateJsonPath","bundle":{"group":"org.apache.nifi","artifact":"nifi-standard-nar","version":"2.10.0"},"properties":{"Destination":"flowfile-attribute","Max String Length":"20 MB","Return Type":"auto-detect","Null Value Representation":"empty string","id":"$.id","Path Not Found Behavior":"ignore"},"propertyDescriptors":{"Destination":{"name":"Destination","displayName":"Destination","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Max String Length":{"name":"Max String Length","displayName":"Max String Length","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Return Type":{"name":"Return Type","displayName":"Return Type","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Null Value Representation":{"name":"Null Value Representation","displayName":"Null Value Representation","identifiesControllerService":false,"sensitive":false,"dynamic":false},"id":{"name":"id","displayName":"id","identifiesControllerService":false,"sensitive":false,"dynamic":true},"Path Not Found Behavior":{"name":"Path Not Found Behavior","displayName":"Path Not Found Behavior","identifiesControllerService":false,"sensitive":false,"dynamic":false}},"style":{},"schedulingPeriod":"0 sec","schedulingStrategy":"TIMER_DRIVEN","executionNode":"ALL","penaltyDuration":"30 sec","yieldDuration":"1 sec","bulletinLevel":"WARN","runDurationMillis":0,"concurrentlySchedulableTaskCount":1,"autoTerminatedRelationships":[],"scheduledState":"ENABLED","retryCount":10,"retriedRelationships":[],"backoffMechanism":"PENALIZE_FLOWFILE","maxBackoffPeriod":"10 mins","componentType":"PROCESSOR","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"22e4a99b-9b48-3dea-920e-e1600becc662","instanceIdentifier":"fb359fe1-f5a9-39b5-94e9-c8dec6112a99","name":"InvokeHTTP","comments":"","position":{"x":-1152.0,"y":-1120.0},"type":"org.apache.nifi.processors.standard.InvokeHTTP","bundle":{"group":"org.apache.nifi","artifact":"nifi-standard-nar","version":"2.10.0"},"properties":{"Request Content-Encoding":"DISABLED","Request Multipart Form-Data Filename Enabled":"true","Request Chunked Transfer-Encoding Enabled":"false","Response Header Request Attributes Prefix":null,"Proxy Configuration Service":null,"HTTP/2 Disabled":"False","Connection Timeout":"15 secs","Response Cookie Strategy":"DISABLED","Socket Read Timeout":"30 secs","Socket Idle Connections":"5","Request Body Enabled":"true","OAuth2 Access Token Refresh Strategy":"ON_TOKEN_EXPIRATION","HTTP URL":"http://ocr-service:8090/api/process","Request OAuth2 Access Token Provider":null,"Socket Idle Timeout":"5 mins","Response Redirects Enabled":"True","Socket Write Timeout":"30 secs","Request Header Attributes Pattern":null,"Response FlowFile Naming Strategy":"RANDOM","Response Cache Enabled":"false","Request Date Header Enabled":"True","Request Failure Penalization Enabled":"false","Response Body Attribute Size":"256","SSL Context Service":null,"Response Generation Required":"false","Request User-Agent":null,"Response Header Request Attributes Enabled":"false","HTTP Method":"POST","Request Username":null,"Request Content-Type":"${mime.type}","Response Body Attribute Name":null,"Request Digest Authentication Enabled":"false","Request Multipart Form-Data Name":null,"Response Cache Size":"10MB","Response Body Ignored":"false"},"propertyDescriptors":{"Request Content-Encoding":{"name":"Request Content-Encoding","displayName":"Request Content-Encoding","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Request Multipart Form-Data Filename Enabled":{"name":"Request Multipart Form-Data Filename Enabled","displayName":"Request Multipart Form-Data Filename Enabled","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Request Chunked Transfer-Encoding Enabled":{"name":"Request Chunked Transfer-Encoding Enabled","displayName":"Request Chunked Transfer-Encoding Enabled","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Response Header Request Attributes Prefix":{"name":"Response Header Request Attributes Prefix","displayName":"Response Header Request Attributes Prefix","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Proxy Configuration Service":{"name":"Proxy Configuration Service","displayName":"Proxy Configuration Service","identifiesControllerService":true,"sensitive":false,"dynamic":false},"HTTP/2 Disabled":{"name":"HTTP/2 Disabled","displayName":"HTTP/2 Disabled","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Connection Timeout":{"name":"Connection Timeout","displayName":"Connection Timeout","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Response Cookie Strategy":{"name":"Response Cookie Strategy","displayName":"Response Cookie Strategy","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Request Password":{"name":"Request Password","displayName":"Request Password","identifiesControllerService":false,"sensitive":true,"dynamic":false},"Socket Read Timeout":{"name":"Socket Read Timeout","displayName":"Socket Read Timeout","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Socket Idle Connections":{"name":"Socket Idle Connections","displayName":"Socket Idle Connections","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Request Body Enabled":{"name":"Request Body Enabled","displayName":"Request Body Enabled","identifiesControllerService":false,"sensitive":false,"dynamic":false},"OAuth2 Access Token Refresh Strategy":{"name":"OAuth2 Access Token Refresh Strategy","displayName":"OAuth2 Access Token Refresh Strategy","identifiesControllerService":false,"sensitive":false,"dynamic":false},"HTTP URL":{"name":"HTTP URL","displayName":"HTTP URL","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Request OAuth2 Access Token Provider":{"name":"Request OAuth2 Access Token Provider","displayName":"Request OAuth2 Access Token Provider","identifiesControllerService":true,"sensitive":false,"dynamic":false},"Socket Idle Timeout":{"name":"Socket Idle Timeout","displayName":"Socket Idle Timeout","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Response Redirects Enabled":{"name":"Response Redirects Enabled","displayName":"Response Redirects Enabled","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Socket Write Timeout":{"name":"Socket Write Timeout","displayName":"Socket Write Timeout","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Request Header Attributes Pattern":{"name":"Request Header Attributes Pattern","displayName":"Request Header Attributes Pattern","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Response FlowFile Naming Strategy":{"name":"Response FlowFile Naming Strategy","displayName":"Response FlowFile Naming Strategy","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Response Cache Enabled":{"name":"Response Cache Enabled","displayName":"Response Cache Enabled","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Request Date Header Enabled":{"name":"Request Date Header Enabled","displayName":"Request Date Header Enabled","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Request Failure Penalization Enabled":{"name":"Request Failure Penalization Enabled","displayName":"Request Failure Penalization Enabled","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Response Body Attribute Size":{"name":"Response Body Attribute Size","displayName":"Response Body Attribute Size","identifiesControllerService":false,"sensitive":false,"dynamic":false},"SSL Context Service":{"name":"SSL Context Service","displayName":"SSL Context Service","identifiesControllerService":true,"sensitive":false,"dynamic":false},"Response Generation Required":{"name":"Response Generation Required","displayName":"Response Generation Required","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Request User-Agent":{"name":"Request User-Agent","displayName":"Request User-Agent","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Response Header Request Attributes Enabled":{"name":"Response Header Request Attributes Enabled","displayName":"Response Header Request Attributes Enabled","identifiesControllerService":false,"sensitive":false,"dynamic":false},"HTTP Method":{"name":"HTTP Method","displayName":"HTTP Method","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Request Username":{"name":"Request Username","displayName":"Request Username","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Request Content-Type":{"name":"Request Content-Type","displayName":"Request Content-Type","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Response Body Attribute Name":{"name":"Response Body Attribute Name","displayName":"Response Body Attribute Name","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Request Digest Authentication Enabled":{"name":"Request Digest Authentication Enabled","displayName":"Request Digest Authentication Enabled","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Request Multipart Form-Data Name":{"name":"Request Multipart Form-Data Name","displayName":"Request Multipart Form-Data Name","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Response Cache Size":{"name":"Response Cache Size","displayName":"Response Cache Size","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Response Body Ignored":{"name":"Response Body Ignored","displayName":"Response Body Ignored","identifiesControllerService":false,"sensitive":false,"dynamic":false}},"style":{},"schedulingPeriod":"0 sec","schedulingStrategy":"TIMER_DRIVEN","executionNode":"ALL","penaltyDuration":"30 sec","yieldDuration":"1 sec","bulletinLevel":"WARN","runDurationMillis":0,"concurrentlySchedulableTaskCount":1,"autoTerminatedRelationships":["Original"],"scheduledState":"ENABLED","retryCount":10,"retriedRelationships":[],"backoffMechanism":"PENALIZE_FLOWFILE","maxBackoffPeriod":"10 mins","componentType":"PROCESSOR","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"97cca146-a5f1-376c-8c4b-a98ca43df6d0","instanceIdentifier":"90324add-4289-37c4-2641-825df420489a","name":"CogStackPrepareRecordForOcr","comments":"","position":{"x":-1152.0,"y":-1320.0},"type":"CogStackPrepareRecordForOcr","bundle":{"group":"org.apache.nifi","artifact":"python-extensions","version":"0.0.1"},"properties":{"binary_field_name":"binarydocument","operation_mode":"base64","process_flow_file_type":"avro","output_text_field_name":"text","document_id_field_name":"id"},"propertyDescriptors":{"binary_field_name":{"name":"binary_field_name","displayName":"binary_field_name","identifiesControllerService":false,"sensitive":false,"dynamic":false},"operation_mode":{"name":"operation_mode","displayName":"operation_mode","identifiesControllerService":false,"sensitive":false,"dynamic":false},"process_flow_file_type":{"name":"process_flow_file_type","displayName":"process_flow_file_type","identifiesControllerService":false,"sensitive":false,"dynamic":false},"output_text_field_name":{"name":"output_text_field_name","displayName":"output_text_field_name","identifiesControllerService":false,"sensitive":false,"dynamic":false},"document_id_field_name":{"name":"document_id_field_name","displayName":"document_id_field_name","identifiesControllerService":false,"sensitive":false,"dynamic":false}},"style":{},"schedulingPeriod":"0 sec","schedulingStrategy":"TIMER_DRIVEN","executionNode":"ALL","penaltyDuration":"30 sec","yieldDuration":"1 sec","bulletinLevel":"WARN","runDurationMillis":25,"concurrentlySchedulableTaskCount":1,"autoTerminatedRelationships":["original"],"scheduledState":"ENABLED","retryCount":10,"retriedRelationships":[],"backoffMechanism":"PENALIZE_FLOWFILE","maxBackoffPeriod":"10 mins","componentType":"PROCESSOR","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"fb1931a4-27ef-3ac3-ab5b-120c720f67e7","instanceIdentifier":"a51ace10-fe83-3670-8178-8e009c932a4e","name":"PutElasticsearchJson","comments":"","position":{"x":-496.0,"y":-912.0},"type":"org.apache.nifi.processors.elasticsearch.PutElasticsearchJson","bundle":{"group":"org.apache.nifi","artifact":"nifi-elasticsearch-restapi-nar","version":"2.10.0"},"properties":{"Dynamic Templates":null,"Treat Not Found as Success":"true","Script":null,"Max FlowFiles Per Batch":"100","Max Batch Size":"","Max JSON Field String Length":"20 MB","Index Field":null,"Output Bulk Request":"false","Index Operation":"index","Index":"${generatetablefetch.tableName}","Input Content Format":"Single JSON","Retain Identifier Field":"true","Log Error Responses":"false","Output Error Responses":"false","Identifier Attribute":"id","Retain Index Field":"true","Type":null,"Suppress Nulls":"never-suppress","Timestamp Field":null,"Scripted Upsert":"false","Identifier Field":null,"Client Service":"06280792-dc54-3d7a-9d35-76ce834dcec3","Character Set":"UTF-8","Retain Timestamp Field":"true"},"propertyDescriptors":{"Dynamic Templates":{"name":"Dynamic Templates","displayName":"Dynamic Templates","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Treat Not Found as Success":{"name":"Treat Not Found as Success","displayName":"Treat Not Found as Success","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Script":{"name":"Script","displayName":"Script","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Max FlowFiles Per Batch":{"name":"Max FlowFiles Per Batch","displayName":"Max FlowFiles Per Batch","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Max Batch Size":{"name":"Max Batch Size","displayName":"Max Batch Size","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Max JSON Field String Length":{"name":"Max JSON Field String Length","displayName":"Max JSON Field String Length","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Index Field":{"name":"Index Field","displayName":"Index Field","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Output Bulk Request":{"name":"Output Bulk Request","displayName":"Output Bulk Request","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Index Operation":{"name":"Index Operation","displayName":"Index Operation","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Index":{"name":"Index","displayName":"Index","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Input Content Format":{"name":"Input Content Format","displayName":"Input Content Format","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Retain Identifier Field":{"name":"Retain Identifier Field","displayName":"Retain Identifier Field","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Log Error Responses":{"name":"Log Error Responses","displayName":"Log Error Responses","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Output Error Responses":{"name":"Output Error Responses","displayName":"Output Error Responses","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Identifier Attribute":{"name":"Identifier Attribute","displayName":"Identifier Attribute","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Retain Index Field":{"name":"Retain Index Field","displayName":"Retain Index Field","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Type":{"name":"Type","displayName":"Type","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Suppress Nulls":{"name":"Suppress Nulls","displayName":"Suppress Nulls","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Timestamp Field":{"name":"Timestamp Field","displayName":"Timestamp Field","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Scripted Upsert":{"name":"Scripted Upsert","displayName":"Scripted Upsert","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Identifier Field":{"name":"Identifier Field","displayName":"Identifier Field","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Client Service":{"name":"Client Service","displayName":"Client Service","identifiesControllerService":true,"sensitive":false,"dynamic":false},"Character Set":{"name":"Character Set","displayName":"Character Set","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Retain Timestamp Field":{"name":"Retain Timestamp Field","displayName":"Retain Timestamp Field","identifiesControllerService":false,"sensitive":false,"dynamic":false}},"style":{},"schedulingPeriod":"0 sec","schedulingStrategy":"TIMER_DRIVEN","executionNode":"ALL","penaltyDuration":"30 sec","yieldDuration":"1 sec","bulletinLevel":"WARN","runDurationMillis":0,"concurrentlySchedulableTaskCount":1,"autoTerminatedRelationships":["original","successful"],"scheduledState":"ENABLED","retryCount":10,"retriedRelationships":[],"backoffMechanism":"PENALIZE_FLOWFILE","maxBackoffPeriod":"10 mins","componentType":"PROCESSOR","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"}],"inputPorts":[],"outputPorts":[],"connections":[{"identifier":"fbabb01c-b959-3437-8bc8-554f3709de45","instanceIdentifier":"836e3567-f8c2-3d1a-ba04-bd6f915710a3","name":"","source":{"id":"63d97939-57b6-36a3-91e2-ee0f4cee2b02","type":"PROCESSOR","groupId":"af2b9237-8c32-355c-9495-3be0e06b4c25","name":"SplitJson","comments":"","instanceIdentifier":"b37a4cb7-8b0d-3a5b-4333-df1160bc8967"},"destination":{"id":"97249984-e607-3cf4-a16c-6dc6f5685b17","type":"FUNNEL","groupId":"af2b9237-8c32-355c-9495-3be0e06b4c25","name":"Funnel","comments":"","instanceIdentifier":"1c8a080c-86a8-3b34-8a9b-2ac2ac3e26d8"},"labelIndex":0,"zIndex":29,"selectedRelationships":["failure"],"backPressureObjectThreshold":10000,"backPressureDataSizeThreshold":"1 GB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","partitioningAttribute":"","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"1bf2e59a-3c46-33c6-809c-d92586c396c8","instanceIdentifier":"ed5619c2-a1e9-3241-8277-773484320537","name":"","source":{"id":"b0a7e1fe-0197-1000-7d26-f460ed2eec5e","type":"PROCESSOR","groupId":"af2b9237-8c32-355c-9495-3be0e06b4c25","name":"GenerateTableFetch-encounters","comments":"","instanceIdentifier":"6ddb9b95-fb84-34e6-c230-f4047af5dee9"},"destination":{"id":"d0e96593-ae93-366d-bda1-17ad8ef2ccd0","type":"FUNNEL","groupId":"af2b9237-8c32-355c-9495-3be0e06b4c25","name":"Funnel","comments":"","instanceIdentifier":"74abb30b-c135-3169-c0d0-61d28a9f8970"},"labelIndex":0,"zIndex":2,"selectedRelationships":["failure"],"backPressureObjectThreshold":10000,"backPressureDataSizeThreshold":"1 GB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","partitioningAttribute":"","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"51f84da6-92b7-3971-9cb4-de65ca7359a8","instanceIdentifier":"7d744319-c9fe-391f-9546-7800d4c4c82a","name":"","source":{"id":"63d97939-57b6-36a3-91e2-ee0f4cee2b02","type":"PROCESSOR","groupId":"af2b9237-8c32-355c-9495-3be0e06b4c25","name":"SplitJson","comments":"","instanceIdentifier":"b37a4cb7-8b0d-3a5b-4333-df1160bc8967"},"destination":{"id":"c1de59a6-f215-3d9b-8597-237a1a50852d","type":"PROCESSOR","groupId":"af2b9237-8c32-355c-9495-3be0e06b4c25","name":"EvaluateJsonPath","comments":"","instanceIdentifier":"4c3e5f3c-eb76-3f7f-3714-792ddd0cfb3a"},"labelIndex":0,"zIndex":28,"selectedRelationships":["split"],"backPressureObjectThreshold":10000,"backPressureDataSizeThreshold":"1 GB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","partitioningAttribute":"","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"84055802-dd6e-3ffe-9aa3-91c092fd4f07","instanceIdentifier":"be2c3c8a-c877-327d-623a-38791b9fe20a","name":"","source":{"id":"fb1931a4-27ef-3ac3-ab5b-120c720f67e7","type":"PROCESSOR","groupId":"af2b9237-8c32-355c-9495-3be0e06b4c25","name":"PutElasticsearchJson","comments":"","instanceIdentifier":"a51ace10-fe83-3670-8178-8e009c932a4e"},"destination":{"id":"5ab77308-8a4c-3b4d-9e2f-681858ce03c4","type":"FUNNEL","groupId":"af2b9237-8c32-355c-9495-3be0e06b4c25","name":"Funnel","comments":"","instanceIdentifier":"c08eb90e-6c7d-3470-ac06-b982c66b6acc"},"labelIndex":0,"zIndex":26,"selectedRelationships":["retry"],"backPressureObjectThreshold":10000,"backPressureDataSizeThreshold":"1 GB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","partitioningAttribute":"","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"410f75b5-207d-3ee1-accc-b5bc768a9b20","instanceIdentifier":"7af46b9a-cf54-3bea-4251-df46bbe1118e","name":"","source":{"id":"c1de59a6-f215-3d9b-8597-237a1a50852d","type":"PROCESSOR","groupId":"af2b9237-8c32-355c-9495-3be0e06b4c25","name":"EvaluateJsonPath","comments":"","instanceIdentifier":"4c3e5f3c-eb76-3f7f-3714-792ddd0cfb3a"},"destination":{"id":"8aa474d8-4037-3e6b-81d7-2f50dbb9b2ee","type":"FUNNEL","groupId":"af2b9237-8c32-355c-9495-3be0e06b4c25","name":"Funnel","comments":"","instanceIdentifier":"711fd6bd-ad04-3a40-71d2-f23913a40548"},"labelIndex":0,"zIndex":33,"selectedRelationships":["unmatched"],"backPressureObjectThreshold":10000,"backPressureDataSizeThreshold":"1 GB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","partitioningAttribute":"","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"ff1c2817-832f-3e6b-a748-ecdf9a3735c6","instanceIdentifier":"74d77ab4-019f-1000-d850-e5d69e720451","name":"","source":{"id":"25d6775d-c9fa-3166-9203-973e735e8877","type":"PROCESSOR","groupId":"af2b9237-8c32-355c-9495-3be0e06b4c25","name":"CogStackParseCogStackServiceResult","comments":"","instanceIdentifier":"74d72c6e-019f-1000-bb5f-e95c500a61f5"},"destination":{"id":"1c96b19b-10b0-3f1f-ab60-be8c2b481516","type":"FUNNEL","groupId":"af2b9237-8c32-355c-9495-3be0e06b4c25","name":"Funnel","comments":"","instanceIdentifier":"6068ef4e-8fda-3edd-0145-90e101066568"},"labelIndex":0,"zIndex":40,"selectedRelationships":["failure"],"backPressureObjectThreshold":10000,"backPressureDataSizeThreshold":"1 GB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","partitioningAttribute":"","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"87d954a3-d65b-347c-b2f4-3fe229c206c9","instanceIdentifier":"cbac13fb-17af-30bd-6882-0b7327e0c83e","name":"","source":{"id":"22e4a99b-9b48-3dea-920e-e1600becc662","type":"PROCESSOR","groupId":"af2b9237-8c32-355c-9495-3be0e06b4c25","name":"InvokeHTTP","comments":"","instanceIdentifier":"fb359fe1-f5a9-39b5-94e9-c8dec6112a99"},"destination":{"id":"25d6775d-c9fa-3166-9203-973e735e8877","type":"PROCESSOR","groupId":"af2b9237-8c32-355c-9495-3be0e06b4c25","name":"CogStackParseCogStackServiceResult","comments":"","instanceIdentifier":"74d72c6e-019f-1000-bb5f-e95c500a61f5"},"labelIndex":0,"zIndex":39,"selectedRelationships":["Response"],"backPressureObjectThreshold":10000,"backPressureDataSizeThreshold":"1 GB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","partitioningAttribute":"","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"7156d11f-9e2f-3bfd-b335-e0aa700950ce","instanceIdentifier":"74d79c2c-019f-1000-bf89-93525c704489","name":"","source":{"id":"25d6775d-c9fa-3166-9203-973e735e8877","type":"PROCESSOR","groupId":"af2b9237-8c32-355c-9495-3be0e06b4c25","name":"CogStackParseCogStackServiceResult","comments":"","instanceIdentifier":"74d72c6e-019f-1000-bb5f-e95c500a61f5"},"destination":{"id":"63d97939-57b6-36a3-91e2-ee0f4cee2b02","type":"PROCESSOR","groupId":"af2b9237-8c32-355c-9495-3be0e06b4c25","name":"SplitJson","comments":"","instanceIdentifier":"b37a4cb7-8b0d-3a5b-4333-df1160bc8967"},"labelIndex":0,"zIndex":41,"selectedRelationships":["success"],"backPressureObjectThreshold":10000,"backPressureDataSizeThreshold":"1 GB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","partitioningAttribute":"","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"94f08f0a-202d-35b7-bee5-5387eb15e2d1","instanceIdentifier":"15c35067-101e-3c43-d101-4a530ff30295","name":"","source":{"id":"b0a7e1fe-0197-1000-7d26-f460ed2eec5e","type":"PROCESSOR","groupId":"af2b9237-8c32-355c-9495-3be0e06b4c25","name":"GenerateTableFetch-encounters","comments":"","instanceIdentifier":"6ddb9b95-fb84-34e6-c230-f4047af5dee9"},"destination":{"id":"b0a7e1fd-0197-1000-7799-44654203e24f","type":"PROCESSOR","groupId":"af2b9237-8c32-355c-9495-3be0e06b4c25","name":"ExecuteSQLRecord","comments":"","instanceIdentifier":"c652a637-5daa-3016-69dc-eb043ae56375"},"labelIndex":0,"zIndex":1,"selectedRelationships":["success"],"backPressureObjectThreshold":10000,"backPressureDataSizeThreshold":"1 GB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","partitioningAttribute":"","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"32dbe6de-d258-37d2-86fe-f6b0626c3384","instanceIdentifier":"1e0a1010-b5ab-3420-ff1a-12383d69982c","name":"","source":{"id":"fb1931a4-27ef-3ac3-ab5b-120c720f67e7","type":"PROCESSOR","groupId":"af2b9237-8c32-355c-9495-3be0e06b4c25","name":"PutElasticsearchJson","comments":"","instanceIdentifier":"a51ace10-fe83-3670-8178-8e009c932a4e"},"destination":{"id":"386800ca-c242-37d0-981a-5d12a5cf0d6e","type":"FUNNEL","groupId":"af2b9237-8c32-355c-9495-3be0e06b4c25","name":"Funnel","comments":"","instanceIdentifier":"413d2c24-9b99-3911-ed11-8ce656a5d68f"},"labelIndex":0,"zIndex":27,"selectedRelationships":["failure","errors"],"backPressureObjectThreshold":10000,"backPressureDataSizeThreshold":"1 GB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","partitioningAttribute":"","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"d525d30e-7744-323c-a6c6-e557aacfef14","instanceIdentifier":"b623aea0-830a-3ccd-db51-9dd74f3cc00b","name":"","source":{"id":"b0a7e1fd-0197-1000-7799-44654203e24f","type":"PROCESSOR","groupId":"af2b9237-8c32-355c-9495-3be0e06b4c25","name":"ExecuteSQLRecord","comments":"","instanceIdentifier":"c652a637-5daa-3016-69dc-eb043ae56375"},"destination":{"id":"97cca146-a5f1-376c-8c4b-a98ca43df6d0","type":"PROCESSOR","groupId":"af2b9237-8c32-355c-9495-3be0e06b4c25","name":"CogStackPrepareRecordForOcr","comments":"","instanceIdentifier":"90324add-4289-37c4-2641-825df420489a"},"labelIndex":0,"zIndex":35,"selectedRelationships":["success"],"backPressureObjectThreshold":10000,"backPressureDataSizeThreshold":"1 GB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","partitioningAttribute":"","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"89389b4e-fe64-360d-94c7-d0ea68444b3c","instanceIdentifier":"c3818a5a-b2ea-3b22-aa1b-09d99a258fa5","name":"","source":{"id":"b0a7e1fd-0197-1000-7799-44654203e24f","type":"PROCESSOR","groupId":"af2b9237-8c32-355c-9495-3be0e06b4c25","name":"ExecuteSQLRecord","comments":"","instanceIdentifier":"c652a637-5daa-3016-69dc-eb043ae56375"},"destination":{"id":"b0dc6924-0197-1000-57e8-8dbd2331558e","type":"FUNNEL","groupId":"af2b9237-8c32-355c-9495-3be0e06b4c25","name":"Funnel","comments":"","instanceIdentifier":"ea2890f4-0312-3463-45fe-9c0495ee0edb"},"labelIndex":0,"zIndex":4,"selectedRelationships":["failure"],"backPressureObjectThreshold":10000,"backPressureDataSizeThreshold":"1 GB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","partitioningAttribute":"","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"6f12409a-5ed1-36d9-b9c4-c82f81a7b571","instanceIdentifier":"1bf02a09-d044-3911-2a2c-8b6b8376bf19","name":"","source":{"id":"c1de59a6-f215-3d9b-8597-237a1a50852d","type":"PROCESSOR","groupId":"af2b9237-8c32-355c-9495-3be0e06b4c25","name":"EvaluateJsonPath","comments":"","instanceIdentifier":"4c3e5f3c-eb76-3f7f-3714-792ddd0cfb3a"},"destination":{"id":"32097907-c0d5-3475-89d9-8801f540a380","type":"FUNNEL","groupId":"af2b9237-8c32-355c-9495-3be0e06b4c25","name":"Funnel","comments":"","instanceIdentifier":"20ece14b-f247-3652-6b4f-93a15b914d8f"},"labelIndex":0,"zIndex":32,"selectedRelationships":["failure"],"backPressureObjectThreshold":10000,"backPressureDataSizeThreshold":"1 GB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","partitioningAttribute":"","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"9e7dfd87-7442-3d32-8cb1-2bdc9fec2838","instanceIdentifier":"49d6a0fe-7c7e-39cf-7207-13e881efab9b","name":"","source":{"id":"22e4a99b-9b48-3dea-920e-e1600becc662","type":"PROCESSOR","groupId":"af2b9237-8c32-355c-9495-3be0e06b4c25","name":"InvokeHTTP","comments":"","instanceIdentifier":"fb359fe1-f5a9-39b5-94e9-c8dec6112a99"},"destination":{"id":"b7c65dca-19eb-3c74-bdfa-a436a086e0cd","type":"FUNNEL","groupId":"af2b9237-8c32-355c-9495-3be0e06b4c25","name":"Funnel","comments":"","instanceIdentifier":"595372fd-5b1d-35b2-b695-d9fde917f226"},"labelIndex":0,"zIndex":8,"selectedRelationships":["No Retry","Retry","Failure"],"backPressureObjectThreshold":10000,"backPressureDataSizeThreshold":"1 GB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","partitioningAttribute":"","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"a7e28e19-fe7a-3362-ab67-0923fd662218","instanceIdentifier":"1bd73264-5323-3125-fc4d-d3b8598f17ba","name":"","source":{"id":"c1de59a6-f215-3d9b-8597-237a1a50852d","type":"PROCESSOR","groupId":"af2b9237-8c32-355c-9495-3be0e06b4c25","name":"EvaluateJsonPath","comments":"","instanceIdentifier":"4c3e5f3c-eb76-3f7f-3714-792ddd0cfb3a"},"destination":{"id":"fb1931a4-27ef-3ac3-ab5b-120c720f67e7","type":"PROCESSOR","groupId":"af2b9237-8c32-355c-9495-3be0e06b4c25","name":"PutElasticsearchJson","comments":"","instanceIdentifier":"a51ace10-fe83-3670-8178-8e009c932a4e"},"labelIndex":0,"zIndex":31,"selectedRelationships":["matched"],"backPressureObjectThreshold":10000,"backPressureDataSizeThreshold":"1 GB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","partitioningAttribute":"","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"97bcd005-0144-370c-9c92-1919663d1032","instanceIdentifier":"932f4afa-687b-37ce-1e2c-e57d638ea841","name":"","source":{"id":"97cca146-a5f1-376c-8c4b-a98ca43df6d0","type":"PROCESSOR","groupId":"af2b9237-8c32-355c-9495-3be0e06b4c25","name":"CogStackPrepareRecordForOcr","comments":"","instanceIdentifier":"90324add-4289-37c4-2641-825df420489a"},"destination":{"id":"22e4a99b-9b48-3dea-920e-e1600becc662","type":"PROCESSOR","groupId":"af2b9237-8c32-355c-9495-3be0e06b4c25","name":"InvokeHTTP","comments":"","instanceIdentifier":"fb359fe1-f5a9-39b5-94e9-c8dec6112a99"},"labelIndex":0,"zIndex":36,"selectedRelationships":["success"],"backPressureObjectThreshold":10000,"backPressureDataSizeThreshold":"1 GB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","partitioningAttribute":"","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"d92ca9c0-f829-307e-83a5-0ad2d8845374","instanceIdentifier":"a3495c8f-cd66-3c38-2921-a98f6dfefe33","name":"","source":{"id":"97cca146-a5f1-376c-8c4b-a98ca43df6d0","type":"PROCESSOR","groupId":"af2b9237-8c32-355c-9495-3be0e06b4c25","name":"CogStackPrepareRecordForOcr","comments":"","instanceIdentifier":"90324add-4289-37c4-2641-825df420489a"},"destination":{"id":"3898e396-9a16-387d-bffa-b1ae83c622e0","type":"FUNNEL","groupId":"af2b9237-8c32-355c-9495-3be0e06b4c25","name":"Funnel","comments":"","instanceIdentifier":"83f0bc54-1645-3ee8-77d9-172120575bc8"},"labelIndex":0,"zIndex":34,"selectedRelationships":["failure"],"backPressureObjectThreshold":10000,"backPressureDataSizeThreshold":"1 GB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","partitioningAttribute":"","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"}],"labels":[{"identifier":"0ccecdf2-914a-32fb-bf20-ea67ea924b9a","instanceIdentifier":"a1e137a9-3e80-3f54-aa4c-2f97dbbcf03f","position":{"x":-792.0,"y":-1120.0},"label":"OCR-ing takes place here, please make sure you have ocr-service/ocr-service-text-only Docker container running, and the ports 8090 (for ocr-ing) / 8091 (text extraction only) are free. Note: if both services are running on the same host as NiFi use port 8090 for both urls: http://ocr-service:8090/api/process or http://ocr-service-text-only:8090/api/process","zIndex":2,"width":296.0,"height":160.0,"style":{"font-size":"12px"},"componentType":"LABEL","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"760c2491-0198-1000-3cfc-b0cebed70256","instanceIdentifier":"7b399dfa-c586-397a-ac9c-21b70cc3e5cd","position":{"x":-1152.0,"y":-544.0},"label":"We split the json because we want 1 record per flowfile, so that we can later update the attributes of the flowfile withe the record's UUID.\n\nIMPORTANT: note the path expression : $.*","zIndex":2,"width":352.0,"height":104.0,"style":{"font-size":"12px"},"componentType":"LABEL","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"30b8928d-679e-3c15-b5b4-ce7efcde9231","instanceIdentifier":"f55a587b-9629-3531-6533-ab166e1aa2e7","position":{"x":-152.0,"y":-1072.0},"label":"Notice that the ElasticsearchJson processor supports only insertion by attribute name, as opposed to the ElasticsearchRecord processor that takes the desired identifier directly from the avro flowfile.","zIndex":4,"width":288.0,"height":136.0,"style":{"font-size":"12px"},"componentType":"LABEL","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"76037ca7-0198-1000-e6d6-7366ac6b8d44","instanceIdentifier":"2e2039b6-e98f-3149-b037-3b3b9908af1a","position":{"x":-1160.0,"y":-1760.0},"label":"Simple ingestion from PgSQL with OCR to ElasticSearch","zIndex":1,"width":368.0,"height":24.0,"style":{"font-size":"12px"},"componentType":"LABEL","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"d1458db0-dccc-3554-98b1-2e54c2937481","instanceIdentifier":"5df7243f-05a2-3500-0966-dc19a63e80a3","position":{"x":-136.0,"y":-584.0},"label":"Check the EvaluateJsonPath, we add a new property called \"id\" with the value \"$.id\" , takes the value from the json.\n","zIndex":3,"width":320.0,"height":104.0,"style":{"font-size":"12px"},"componentType":"LABEL","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"}],"funnels":[{"identifier":"32097907-c0d5-3475-89d9-8801f540a380","instanceIdentifier":"20ece14b-f247-3652-6b4f-93a15b914d8f","position":{"x":-344.0,"y":-496.0},"componentType":"FUNNEL","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"b0dc6924-0197-1000-57e8-8dbd2331558e","instanceIdentifier":"ea2890f4-0312-3463-45fe-9c0495ee0edb","position":{"x":-1480.0,"y":-1488.0},"componentType":"FUNNEL","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"d0e96593-ae93-366d-bda1-17ad8ef2ccd0","instanceIdentifier":"74abb30b-c135-3169-c0d0-61d28a9f8970","position":{"x":-1480.0,"y":-1680.0},"componentType":"FUNNEL","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"97249984-e607-3cf4-a16c-6dc6f5685b17","instanceIdentifier":"1c8a080c-86a8-3b34-8a9b-2ac2ac3e26d8","position":{"x":-1480.0,"y":-640.0},"componentType":"FUNNEL","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"3898e396-9a16-387d-bffa-b1ae83c622e0","instanceIdentifier":"83f0bc54-1645-3ee8-77d9-172120575bc8","position":{"x":-1480.0,"y":-1280.0},"componentType":"FUNNEL","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"8aa474d8-4037-3e6b-81d7-2f50dbb9b2ee","instanceIdentifier":"711fd6bd-ad04-3a40-71d2-f23913a40548","position":{"x":136.0,"y":-656.0},"componentType":"FUNNEL","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"386800ca-c242-37d0-981a-5d12a5cf0d6e","instanceIdentifier":"413d2c24-9b99-3911-ed11-8ce656a5d68f","position":{"x":-344.0,"y":-1032.0},"componentType":"FUNNEL","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"b7c65dca-19eb-3c74-bdfa-a436a086e0cd","instanceIdentifier":"595372fd-5b1d-35b2-b695-d9fde917f226","position":{"x":-1480.0,"y":-1080.0},"componentType":"FUNNEL","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"1c96b19b-10b0-3f1f-ab60-be8c2b481516","instanceIdentifier":"6068ef4e-8fda-3edd-0145-90e101066568","position":{"x":-1480.0,"y":-856.0},"componentType":"FUNNEL","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"5ab77308-8a4c-3b4d-9e2f-681858ce03c4","instanceIdentifier":"c08eb90e-6c7d-3470-ac06-b982c66b6acc","position":{"x":136.0,"y":-864.0},"componentType":"FUNNEL","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"}],"controllerServices":[{"identifier":"cf502be1-705e-3731-95b7-2dbd5c603c26","instanceIdentifier":"4b0cd571-6a65-3785-18cd-7f92468e85f4","name":"StandardSSLContextService","comments":"","type":"org.apache.nifi.ssl.StandardSSLContextService","bundle":{"group":"org.apache.nifi","artifact":"nifi-ssl-context-service-nar","version":"2.10.0"},"properties":{"Truststore Type":"JKS","Keystore Type":"JKS","Truststore Filename":"/security/certificates/elastic/opensearch/elasticsearch/elasticsearch-1/elasticsearch-1-truststore.key","TLS Protocol":"TLS","Keystore Filename":"/security/certificates/elastic/opensearch/elasticsearch/elasticsearch-1/elasticsearch-1-keystore.jks"},"propertyDescriptors":{"Truststore Type":{"name":"Truststore Type","displayName":"Truststore Type","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Key Password":{"name":"Key Password","displayName":"Key Password","identifiesControllerService":false,"sensitive":true,"dynamic":false},"Keystore Type":{"name":"Keystore Type","displayName":"Keystore Type","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Truststore Filename":{"name":"Truststore Filename","displayName":"Truststore Filename","identifiesControllerService":false,"sensitive":false,"dynamic":false,"resourceDefinition":{"cardinality":"SINGLE","resourceTypes":["FILE"]}},"Keystore Password":{"name":"Keystore Password","displayName":"Keystore Password","identifiesControllerService":false,"sensitive":true,"dynamic":false},"Truststore Password":{"name":"Truststore Password","displayName":"Truststore Password","identifiesControllerService":false,"sensitive":true,"dynamic":false},"TLS Protocol":{"name":"TLS Protocol","displayName":"TLS Protocol","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Keystore Filename":{"name":"Keystore Filename","displayName":"Keystore Filename","identifiesControllerService":false,"sensitive":false,"dynamic":false,"resourceDefinition":{"cardinality":"SINGLE","resourceTypes":["FILE"]}}},"controllerServiceApis":[{"type":"org.apache.nifi.ssl.SSLContextProvider","bundle":{"group":"org.apache.nifi","artifact":"nifi-standard-services-api-nar","version":"2.10.0"}},{"type":"org.apache.nifi.ssl.SSLContextService","bundle":{"group":"org.apache.nifi","artifact":"nifi-standard-services-api-nar","version":"2.10.0"}}],"scheduledState":"DISABLED","bulletinLevel":"WARN","componentType":"CONTROLLER_SERVICE","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"ae4d900f-d06e-3cc3-acc5-4f6f222a5064","instanceIdentifier":"62a09a84-3bee-3ffd-b5d0-031d1b7e8cee","name":"DBCPConnectionPool","comments":"","type":"org.apache.nifi.dbcp.DBCPConnectionPool","bundle":{"group":"org.apache.nifi","artifact":"nifi-dbcp-service-nar","version":"2.10.0"},"properties":{"Maximum Idle Connections":"8","Database Driver Locations":"/opt/nifi/drivers/postgresql-42.7.7.jar","Max Wait Time":"500 millis","Database Driver Class Name":"org.postgresql.Driver","Minimum Idle Connections":"0","Kerberos User Service":null,"Max Total Connections":"8","Validation Query":null,"Database Password Provider":null,"Database Connection URL":"${POSTGRES_SAMPLES_JDBC_URL}","Password Source":"PASSWORD","Time Between Eviction Runs":"-1","Database User":"${POSTGRES_INGEST_USER_SAMPLES}","Password":"${POSTGRES_INGEST_PASSWORD_SAMPLES}","Soft Minimum Evictable Idle Time":"-1","Maximum Connection Lifetime":"-1","Minimum Evictable Idle Time":"30 mins"},"propertyDescriptors":{"Maximum Idle Connections":{"name":"Maximum Idle Connections","displayName":"Maximum Idle Connections","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Database Driver Locations":{"name":"Database Driver Locations","displayName":"Database Driver Locations","identifiesControllerService":false,"sensitive":false,"dynamic":false,"resourceDefinition":{"cardinality":"MULTIPLE","resourceTypes":["FILE","DIRECTORY","URL"]}},"Max Wait Time":{"name":"Max Wait Time","displayName":"Max Wait Time","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Database Driver Class Name":{"name":"Database Driver Class Name","displayName":"Database Driver Class Name","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Minimum Idle Connections":{"name":"Minimum Idle Connections","displayName":"Minimum Idle Connections","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Kerberos User Service":{"name":"Kerberos User Service","displayName":"Kerberos User Service","identifiesControllerService":true,"sensitive":false,"dynamic":false},"Max Total Connections":{"name":"Max Total Connections","displayName":"Max Total Connections","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Validation Query":{"name":"Validation Query","displayName":"Validation Query","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Database Password Provider":{"name":"Database Password Provider","displayName":"Database Password Provider","identifiesControllerService":true,"sensitive":false,"dynamic":false},"Database Connection URL":{"name":"Database Connection URL","displayName":"Database Connection URL","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Password Source":{"name":"Password Source","displayName":"Password Source","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Time Between Eviction Runs":{"name":"Time Between Eviction Runs","displayName":"Time Between Eviction Runs","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Database User":{"name":"Database User","displayName":"Database User","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Soft Minimum Evictable Idle Time":{"name":"Soft Minimum Evictable Idle Time","displayName":"Soft Minimum Evictable Idle Time","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Maximum Connection Lifetime":{"name":"Maximum Connection Lifetime","displayName":"Maximum Connection Lifetime","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Minimum Evictable Idle Time":{"name":"Minimum Evictable Idle Time","displayName":"Minimum Evictable Idle Time","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Password":{"name":"Password","displayName":"Password","identifiesControllerService":false,"sensitive":true,"dynamic":false}},"controllerServiceApis":[{"type":"org.apache.nifi.dbcp.DBCPService","bundle":{"group":"org.apache.nifi","artifact":"nifi-standard-services-api-nar","version":"2.10.0"}}],"scheduledState":"DISABLED","bulletinLevel":"WARN","componentType":"CONTROLLER_SERVICE","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"06280792-dc54-3d7a-9d35-76ce834dcec3","instanceIdentifier":"5418c689-1502-3458-f583-b82f274a01ac","name":"ElasticSearchClientServiceImpl","comments":"","type":"org.apache.nifi.elasticsearch.ElasticSearchClientServiceImpl","bundle":{"group":"org.apache.nifi","artifact":"nifi-elasticsearch-client-service-nar","version":"2.10.0"},"properties":{"Sniffer Request Timeout":"1 sec","Connect Timeout":"5000","HTTP Hosts":"https://elasticsearch-1:9200","Path Prefix":null,"API Key ID":null,"Enable Compression":"false","Read Timeout":"60000","Sniff on Failure":"false","Sniffer Failure Delay":"1 min","OAuth2 Access Token Provider":null,"Suppress Null and Empty Values":"always-suppress","Node Selector":"ANY","Proxy Configuration Service":null,"Strict Deprecation":"false","Sniff Cluster Nodes":"false","SSL Context Service":"cf502be1-705e-3731-95b7-2dbd5c603c26","Send Meta Header":"true","Username":"admin","Authorization Scheme":"BASIC","Sniffer Interval":"5 mins","Run As User":null,"Character Set":"UTF-8"},"propertyDescriptors":{"Sniffer Request Timeout":{"name":"Sniffer Request Timeout","displayName":"Sniffer Request Timeout","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Connect Timeout":{"name":"Connect Timeout","displayName":"Connect Timeout","identifiesControllerService":false,"sensitive":false,"dynamic":false},"API Key ID":{"name":"API Key ID","displayName":"API Key ID","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Enable Compression":{"name":"Enable Compression","displayName":"Enable Compression","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Sniff on Failure":{"name":"Sniff on Failure","displayName":"Sniff on Failure","identifiesControllerService":false,"sensitive":false,"dynamic":false},"OAuth2 Access Token Provider":{"name":"OAuth2 Access Token Provider","displayName":"OAuth2 Access Token Provider","identifiesControllerService":true,"sensitive":false,"dynamic":false},"Node Selector":{"name":"Node Selector","displayName":"Node Selector","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Proxy Configuration Service":{"name":"Proxy Configuration Service","displayName":"Proxy Configuration Service","identifiesControllerService":true,"sensitive":false,"dynamic":false},"Strict Deprecation":{"name":"Strict Deprecation","displayName":"Strict Deprecation","identifiesControllerService":false,"sensitive":false,"dynamic":false},"JWT Shared Secret":{"name":"JWT Shared Secret","displayName":"JWT Shared Secret","identifiesControllerService":false,"sensitive":true,"dynamic":false},"API Key":{"name":"API Key","displayName":"API Key","identifiesControllerService":false,"sensitive":true,"dynamic":false},"Send Meta Header":{"name":"Send Meta Header","displayName":"Send Meta Header","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Authorization Scheme":{"name":"Authorization Scheme","displayName":"Authorization Scheme","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Password":{"name":"Password","displayName":"Password","identifiesControllerService":false,"sensitive":true,"dynamic":false},"HTTP Hosts":{"name":"HTTP Hosts","displayName":"HTTP Hosts","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Path Prefix":{"name":"Path Prefix","displayName":"Path Prefix","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Read Timeout":{"name":"Read Timeout","displayName":"Read Timeout","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Sniffer Failure Delay":{"name":"Sniffer Failure Delay","displayName":"Sniffer Failure Delay","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Suppress Null and Empty Values":{"name":"Suppress Null and Empty Values","displayName":"Suppress Null and Empty Values","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Sniff Cluster Nodes":{"name":"Sniff Cluster Nodes","displayName":"Sniff Cluster Nodes","identifiesControllerService":false,"sensitive":false,"dynamic":false},"SSL Context Service":{"name":"SSL Context Service","displayName":"SSL Context Service","identifiesControllerService":true,"sensitive":false,"dynamic":false},"Username":{"name":"Username","displayName":"Username","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Sniffer Interval":{"name":"Sniffer Interval","displayName":"Sniffer Interval","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Run As User":{"name":"Run As User","displayName":"Run As User","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Character Set":{"name":"Character Set","displayName":"Character Set","identifiesControllerService":false,"sensitive":false,"dynamic":false}},"controllerServiceApis":[{"type":"org.apache.nifi.elasticsearch.ElasticSearchClientService","bundle":{"group":"org.apache.nifi","artifact":"nifi-elasticsearch-client-service-api-nar","version":"2.10.0"}}],"scheduledState":"DISABLED","bulletinLevel":"WARN","componentType":"CONTROLLER_SERVICE","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"},{"identifier":"4ad0bf71-a5de-39f7-9f74-07213d115a9f","instanceIdentifier":"8411d042-d05f-3bf6-7f0e-2c12cf8ed007","name":"AvroRecordSetWriter","comments":"","type":"org.apache.nifi.avro.AvroRecordSetWriter","bundle":{"group":"org.apache.nifi","artifact":"nifi-record-serialization-services-nar","version":"2.10.0"},"properties":{"Schema Branch":null,"Schema Reference Reader":null,"Schema Text":"${avro.schema}","Schema Registry":null,"Schema Access Strategy":"inherit-record-schema","Schema Name":"${schema.name}","Schema Write Strategy":"avro-embedded","Encoder Pool Size":"32","Compression Format":"NONE","Schema Cache":null,"Schema Version":null,"Cache Size":"1000","Schema Reference Writer":null},"propertyDescriptors":{"Schema Branch":{"name":"Schema Branch","displayName":"Schema Branch","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Schema Reference Reader":{"name":"Schema Reference Reader","displayName":"Schema Reference Reader","identifiesControllerService":true,"sensitive":false,"dynamic":false},"Schema Text":{"name":"Schema Text","displayName":"Schema Text","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Schema Registry":{"name":"Schema Registry","displayName":"Schema Registry","identifiesControllerService":true,"sensitive":false,"dynamic":false},"Schema Access Strategy":{"name":"Schema Access Strategy","displayName":"Schema Access Strategy","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Schema Name":{"name":"Schema Name","displayName":"Schema Name","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Schema Write Strategy":{"name":"Schema Write Strategy","displayName":"Schema Write Strategy","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Encoder Pool Size":{"name":"Encoder Pool Size","displayName":"Encoder Pool Size","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Compression Format":{"name":"Compression Format","displayName":"Compression Format","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Schema Cache":{"name":"Schema Cache","displayName":"Schema Cache","identifiesControllerService":true,"sensitive":false,"dynamic":false},"Schema Version":{"name":"Schema Version","displayName":"Schema Version","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Cache Size":{"name":"Cache Size","displayName":"Cache Size","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Schema Reference Writer":{"name":"Schema Reference Writer","displayName":"Schema Reference Writer","identifiesControllerService":true,"sensitive":false,"dynamic":false}},"controllerServiceApis":[{"type":"org.apache.nifi.serialization.RecordSetWriterFactory","bundle":{"group":"org.apache.nifi","artifact":"nifi-standard-services-api-nar","version":"2.10.0"}}],"scheduledState":"DISABLED","bulletinLevel":"WARN","componentType":"CONTROLLER_SERVICE","groupIdentifier":"af2b9237-8c32-355c-9495-3be0e06b4c25"}],"defaultFlowFileExpiration":"0 sec","defaultBackPressureObjectThreshold":10000,"defaultBackPressureDataSizeThreshold":"1 GB","scheduledState":"ENABLED","executionEngine":"INHERITED","maxConcurrentTasks":1,"statelessFlowTimeout":"1 min","flowFileConcurrency":"UNBOUNDED","flowFileOutboundPolicy":"STREAM_WHEN_AVAILABLE","componentType":"PROCESS_GROUP"},"externalControllerServices":{},"parameterContexts":{},"flowEncodingVersion":"1.0","parameterProviders":{},"latest":false} \ No newline at end of file diff --git a/nifi/user_templates/opensearch_ingest_docs_db_to_es.json b/nifi/user_templates/opensearch_ingest_docs_db_to_es.json index f3b22c7b7..2aa9f11a2 100644 --- a/nifi/user_templates/opensearch_ingest_docs_db_to_es.json +++ b/nifi/user_templates/opensearch_ingest_docs_db_to_es.json @@ -1 +1 @@ -{"flowContents":{"identifier":"62dd0a2b-528e-3db2-af01-43a69b2b4110","instanceIdentifier":"1d75b764-019f-1000-b3ce-e679959f9af7","name":"opensearch_ingest_docs_db_to_es","comments":"","position":{"x":384.0,"y":347.0},"processGroups":[],"remoteProcessGroups":[],"processors":[{"identifier":"9496f814-758e-3ff4-be63-09cfd4e2a7b5","instanceIdentifier":"9a73d2f8-fae5-314f-a043-96aa082f7ab5","name":"ExecuteSQLRecord","comments":"","position":{"x":-608.0,"y":-496.0},"type":"org.apache.nifi.processors.standard.ExecuteSQLRecord","bundle":{"group":"org.apache.nifi","artifact":"nifi-standard-nar","version":"2.10.0"},"properties":{"Set Auto Commit":"true","Max Wait Time":"0 seconds","SQL Query":null,"Normalize Table/Column Names":"false","Database Connection Pooling Service":"ae4d900f-d06e-3cc3-acc5-4f6f222a5064","Default Decimal Scale":"0","SQL Post-Query":null,"Fetch Size":"0","Output Batch Size":"0","Use Avro Logical Types":"true","Record Writer":"4ad0bf71-a5de-39f7-9f74-07213d115a9f","SQL Pre-Query":null,"Default Decimal Precision":"10","Max Rows Per FlowFile":"0"},"propertyDescriptors":{"Set Auto Commit":{"name":"Set Auto Commit","displayName":"Set Auto Commit","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Max Wait Time":{"name":"Max Wait Time","displayName":"Max Wait Time","identifiesControllerService":false,"sensitive":false,"dynamic":false},"SQL Query":{"name":"SQL Query","displayName":"SQL Query","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Normalize Table/Column Names":{"name":"Normalize Table/Column Names","displayName":"Normalize Table/Column Names","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Database Connection Pooling Service":{"name":"Database Connection Pooling Service","displayName":"Database Connection Pooling Service","identifiesControllerService":true,"sensitive":false,"dynamic":false},"Default Decimal Scale":{"name":"Default Decimal Scale","displayName":"Default Decimal Scale","identifiesControllerService":false,"sensitive":false,"dynamic":false},"SQL Post-Query":{"name":"SQL Post-Query","displayName":"SQL Post-Query","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Fetch Size":{"name":"Fetch Size","displayName":"Fetch Size","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Output Batch Size":{"name":"Output Batch Size","displayName":"Output Batch Size","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Use Avro Logical Types":{"name":"Use Avro Logical Types","displayName":"Use Avro Logical Types","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Record Writer":{"name":"Record Writer","displayName":"Record Writer","identifiesControllerService":true,"sensitive":false,"dynamic":false},"SQL Pre-Query":{"name":"SQL Pre-Query","displayName":"SQL Pre-Query","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Default Decimal Precision":{"name":"Default Decimal Precision","displayName":"Default Decimal Precision","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Max Rows Per FlowFile":{"name":"Max Rows Per FlowFile","displayName":"Max Rows Per FlowFile","identifiesControllerService":false,"sensitive":false,"dynamic":false}},"style":{},"schedulingPeriod":"0 sec","schedulingStrategy":"TIMER_DRIVEN","executionNode":"ALL","penaltyDuration":"30 sec","yieldDuration":"1 sec","bulletinLevel":"WARN","runDurationMillis":0,"concurrentlySchedulableTaskCount":1,"autoTerminatedRelationships":[],"scheduledState":"ENABLED","retryCount":10,"retriedRelationships":[],"backoffMechanism":"PENALIZE_FLOWFILE","maxBackoffPeriod":"10 mins","componentType":"PROCESSOR","groupIdentifier":"62dd0a2b-528e-3db2-af01-43a69b2b4110"},{"identifier":"9c434759-d273-371e-a0dc-6e1f770d6a98","instanceIdentifier":"7cbd2dba-4532-3bb2-2ca0-cc42e36294af","name":"GenerateTableFetch-medical_reports_text","comments":"","position":{"x":-608.0,"y":-688.0},"type":"org.apache.nifi.processors.standard.GenerateTableFetch","bundle":{"group":"org.apache.nifi","artifact":"nifi-standard-nar","version":"2.10.0"},"properties":{"Table Name":"medical_reports_text","Max Wait Time":"0 seconds","Partition Size":"10000","Database Connection Pooling Service":"ae4d900f-d06e-3cc3-acc5-4f6f222a5064","Database Type":"PostgreSQL","Columns to Return":"sampleid,typeid,dct,filename,\"document\",CAST(docid AS VARCHAR(255)) AS docid","Maximum-value Columns":"docid","Custom ORDER BY Column":null,"Output Empty FlowFile on Zero Results":"false","Database Dialect Service":null,"Column for Value Partitioning":"docid","Additional WHERE Clause":null},"propertyDescriptors":{"Table Name":{"name":"Table Name","displayName":"Table Name","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Max Wait Time":{"name":"Max Wait Time","displayName":"Max Wait Time","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Partition Size":{"name":"Partition Size","displayName":"Partition Size","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Database Connection Pooling Service":{"name":"Database Connection Pooling Service","displayName":"Database Connection Pooling Service","identifiesControllerService":true,"sensitive":false,"dynamic":false},"Database Type":{"name":"Database Type","displayName":"Database Type","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Columns to Return":{"name":"Columns to Return","displayName":"Columns to Return","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Maximum-value Columns":{"name":"Maximum-value Columns","displayName":"Maximum-value Columns","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Custom ORDER BY Column":{"name":"Custom ORDER BY Column","displayName":"Custom ORDER BY Column","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Output Empty FlowFile on Zero Results":{"name":"Output Empty FlowFile on Zero Results","displayName":"Output Empty FlowFile on Zero Results","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Database Dialect Service":{"name":"Database Dialect Service","displayName":"Database Dialect Service","identifiesControllerService":true,"sensitive":false,"dynamic":false},"Column for Value Partitioning":{"name":"Column for Value Partitioning","displayName":"Column for Value Partitioning","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Additional WHERE Clause":{"name":"Additional WHERE Clause","displayName":"Additional WHERE Clause","identifiesControllerService":false,"sensitive":false,"dynamic":false}},"style":{},"schedulingPeriod":"0 sec","schedulingStrategy":"TIMER_DRIVEN","executionNode":"ALL","penaltyDuration":"30 sec","yieldDuration":"1 sec","bulletinLevel":"WARN","runDurationMillis":0,"concurrentlySchedulableTaskCount":1,"autoTerminatedRelationships":[],"scheduledState":"ENABLED","retryCount":10,"retriedRelationships":[],"backoffMechanism":"PENALIZE_FLOWFILE","maxBackoffPeriod":"10 mins","componentType":"PROCESSOR","groupIdentifier":"62dd0a2b-528e-3db2-af01-43a69b2b4110"},{"identifier":"ae73dcb2-f24b-3e7a-911d-a7b3c7e0678e","instanceIdentifier":"40a6e665-bf5d-346a-ea03-ef647cd8f90f","name":"PutElasticsearchRecord","comments":"","position":{"x":-608.0,"y":-288.0},"type":"org.apache.nifi.processors.elasticsearch.PutElasticsearchRecord","bundle":{"group":"org.apache.nifi","artifact":"nifi-elasticsearch-restapi-nar","version":"2.10.0"},"properties":{"Timestamp Value":null,"Max JSON Field String Length":"64 MB","Dynamic Templates Record Path":null,"Index Operation":"index","Index":"${generatetablefetch.tableName}","Record Reader":"384cff7c-0232-3e2c-bc72-37e5bdc266bc","Date Format":null,"Retain Record Path ID Field":"false","Time Format":null,"Result Record Writer":"bb71c362-4fc9-314e-96ed-50c123266093","Treat Not Found as Success":"true","Group Results by Bulk Error Type":"false","Scripted Upsert Record Path":null,"Index Operation Record Path":null,"Type Record Path":null,"ID Record Path":"/docid","Log Error Responses":"false","Script Record Path":null,"Output Error Responses":"false","Retain Record Timestamp":"false","Timestamp Format":null,"Type":null,"Batch Size":"10000","Client Service":"06280792-dc54-3d7a-9d35-76ce834dcec3","Index Record Path":null,"Timestamp Record Path":null},"propertyDescriptors":{"Timestamp Value":{"name":"Timestamp Value","displayName":"Timestamp Value","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Max JSON Field String Length":{"name":"Max JSON Field String Length","displayName":"Max JSON Field String Length","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Dynamic Templates Record Path":{"name":"Dynamic Templates Record Path","displayName":"Dynamic Templates Record Path","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Index Operation":{"name":"Index Operation","displayName":"Index Operation","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Index":{"name":"Index","displayName":"Index","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Record Reader":{"name":"Record Reader","displayName":"Record Reader","identifiesControllerService":true,"sensitive":false,"dynamic":false},"Date Format":{"name":"Date Format","displayName":"Date Format","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Retain Record Path ID Field":{"name":"Retain Record Path ID Field","displayName":"Retain Record Path ID Field","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Time Format":{"name":"Time Format","displayName":"Time Format","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Result Record Writer":{"name":"Result Record Writer","displayName":"Result Record Writer","identifiesControllerService":true,"sensitive":false,"dynamic":false},"Treat Not Found as Success":{"name":"Treat Not Found as Success","displayName":"Treat Not Found as Success","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Group Results by Bulk Error Type":{"name":"Group Results by Bulk Error Type","displayName":"Group Results by Bulk Error Type","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Scripted Upsert Record Path":{"name":"Scripted Upsert Record Path","displayName":"Scripted Upsert Record Path","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Index Operation Record Path":{"name":"Index Operation Record Path","displayName":"Index Operation Record Path","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Type Record Path":{"name":"Type Record Path","displayName":"Type Record Path","identifiesControllerService":false,"sensitive":false,"dynamic":false},"ID Record Path":{"name":"ID Record Path","displayName":"ID Record Path","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Log Error Responses":{"name":"Log Error Responses","displayName":"Log Error Responses","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Script Record Path":{"name":"Script Record Path","displayName":"Script Record Path","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Output Error Responses":{"name":"Output Error Responses","displayName":"Output Error Responses","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Retain Record Timestamp":{"name":"Retain Record Timestamp","displayName":"Retain Record Timestamp","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Timestamp Format":{"name":"Timestamp Format","displayName":"Timestamp Format","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Type":{"name":"Type","displayName":"Type","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Batch Size":{"name":"Batch Size","displayName":"Batch Size","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Client Service":{"name":"Client Service","displayName":"Client Service","identifiesControllerService":true,"sensitive":false,"dynamic":false},"Index Record Path":{"name":"Index Record Path","displayName":"Index Record Path","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Timestamp Record Path":{"name":"Timestamp Record Path","displayName":"Timestamp Record Path","identifiesControllerService":false,"sensitive":false,"dynamic":false}},"style":{},"schedulingPeriod":"0 sec","schedulingStrategy":"TIMER_DRIVEN","executionNode":"ALL","penaltyDuration":"30 sec","yieldDuration":"1 sec","bulletinLevel":"WARN","runDurationMillis":0,"concurrentlySchedulableTaskCount":1,"autoTerminatedRelationships":["original","successful"],"scheduledState":"ENABLED","retryCount":10,"retriedRelationships":[],"backoffMechanism":"PENALIZE_FLOWFILE","maxBackoffPeriod":"10 mins","componentType":"PROCESSOR","groupIdentifier":"62dd0a2b-528e-3db2-af01-43a69b2b4110"}],"inputPorts":[],"outputPorts":[],"connections":[{"identifier":"992d8825-b10c-3844-9ba5-5076f20673b7","instanceIdentifier":"91e284b1-60de-3e49-0424-a24a3f839e97","name":"","source":{"id":"9c434759-d273-371e-a0dc-6e1f770d6a98","type":"PROCESSOR","groupId":"62dd0a2b-528e-3db2-af01-43a69b2b4110","name":"GenerateTableFetch-medical_reports_text","comments":"","instanceIdentifier":"7cbd2dba-4532-3bb2-2ca0-cc42e36294af"},"destination":{"id":"9496f814-758e-3ff4-be63-09cfd4e2a7b5","type":"PROCESSOR","groupId":"62dd0a2b-528e-3db2-af01-43a69b2b4110","name":"ExecuteSQLRecord","comments":"","instanceIdentifier":"9a73d2f8-fae5-314f-a043-96aa082f7ab5"},"labelIndex":0,"zIndex":1,"selectedRelationships":["success"],"backPressureObjectThreshold":10000,"backPressureDataSizeThreshold":"1 GB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","partitioningAttribute":"","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"62dd0a2b-528e-3db2-af01-43a69b2b4110"},{"identifier":"4daeaa34-b854-324a-b970-4878b7095d0f","instanceIdentifier":"64b44b06-87c1-3aaf-00e6-43241d85aad8","name":"","source":{"id":"9496f814-758e-3ff4-be63-09cfd4e2a7b5","type":"PROCESSOR","groupId":"62dd0a2b-528e-3db2-af01-43a69b2b4110","name":"ExecuteSQLRecord","comments":"","instanceIdentifier":"9a73d2f8-fae5-314f-a043-96aa082f7ab5"},"destination":{"id":"518a227f-0197-1000-c916-878366970718","type":"FUNNEL","groupId":"62dd0a2b-528e-3db2-af01-43a69b2b4110","name":"Funnel","comments":"","instanceIdentifier":"932fe33f-692b-39b5-9fae-c69ac9c23fe9"},"labelIndex":0,"zIndex":3,"selectedRelationships":["failure"],"backPressureObjectThreshold":10000,"backPressureDataSizeThreshold":"1 GB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","partitioningAttribute":"","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"62dd0a2b-528e-3db2-af01-43a69b2b4110"},{"identifier":"bf5a6660-9dd9-3efc-bdb3-fab678070664","instanceIdentifier":"60223890-f9c7-3291-6cd6-9ae237099fe9","name":"","source":{"id":"9496f814-758e-3ff4-be63-09cfd4e2a7b5","type":"PROCESSOR","groupId":"62dd0a2b-528e-3db2-af01-43a69b2b4110","name":"ExecuteSQLRecord","comments":"","instanceIdentifier":"9a73d2f8-fae5-314f-a043-96aa082f7ab5"},"destination":{"id":"ae73dcb2-f24b-3e7a-911d-a7b3c7e0678e","type":"PROCESSOR","groupId":"62dd0a2b-528e-3db2-af01-43a69b2b4110","name":"PutElasticsearchRecord","comments":"","instanceIdentifier":"40a6e665-bf5d-346a-ea03-ef647cd8f90f"},"labelIndex":0,"zIndex":4,"selectedRelationships":["success"],"backPressureObjectThreshold":10000,"backPressureDataSizeThreshold":"1 GB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","partitioningAttribute":"","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"62dd0a2b-528e-3db2-af01-43a69b2b4110"},{"identifier":"d697dfd0-3e44-385a-a55c-5b52403c9903","instanceIdentifier":"6508ef68-6f8d-38ca-aaa7-21521e910031","name":"","source":{"id":"ae73dcb2-f24b-3e7a-911d-a7b3c7e0678e","type":"PROCESSOR","groupId":"62dd0a2b-528e-3db2-af01-43a69b2b4110","name":"PutElasticsearchRecord","comments":"","instanceIdentifier":"40a6e665-bf5d-346a-ea03-ef647cd8f90f"},"destination":{"id":"518b162d-0197-1000-931b-72a60e7fcb87","type":"FUNNEL","groupId":"62dd0a2b-528e-3db2-af01-43a69b2b4110","name":"Funnel","comments":"","instanceIdentifier":"331295b4-db6e-3fbc-ff90-eb89a70d717b"},"labelIndex":0,"zIndex":6,"selectedRelationships":["retry"],"backPressureObjectThreshold":10000,"backPressureDataSizeThreshold":"1 GB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","partitioningAttribute":"","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"62dd0a2b-528e-3db2-af01-43a69b2b4110"},{"identifier":"f24c9db6-764c-3075-a921-8f5a3b5a1d29","instanceIdentifier":"75e50475-b018-32a4-a3ae-b6e571b70855","name":"","source":{"id":"ae73dcb2-f24b-3e7a-911d-a7b3c7e0678e","type":"PROCESSOR","groupId":"62dd0a2b-528e-3db2-af01-43a69b2b4110","name":"PutElasticsearchRecord","comments":"","instanceIdentifier":"40a6e665-bf5d-346a-ea03-ef647cd8f90f"},"destination":{"id":"518aca05-0197-1000-00c9-e6e151102f62","type":"FUNNEL","groupId":"62dd0a2b-528e-3db2-af01-43a69b2b4110","name":"Funnel","comments":"","instanceIdentifier":"5bc9b53c-d3de-3ef5-6186-faee77d5f3cf"},"labelIndex":0,"zIndex":5,"selectedRelationships":["failure","errors"],"backPressureObjectThreshold":10000,"backPressureDataSizeThreshold":"1 GB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","partitioningAttribute":"","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"62dd0a2b-528e-3db2-af01-43a69b2b4110"},{"identifier":"ba753c99-d884-3eef-8618-8fd11a022a32","instanceIdentifier":"6afb924d-728a-3d11-4746-e6ebae4a896c","name":"","source":{"id":"9c434759-d273-371e-a0dc-6e1f770d6a98","type":"PROCESSOR","groupId":"62dd0a2b-528e-3db2-af01-43a69b2b4110","name":"GenerateTableFetch-medical_reports_text","comments":"","instanceIdentifier":"7cbd2dba-4532-3bb2-2ca0-cc42e36294af"},"destination":{"id":"a8b3cffd-32c7-33c3-bf00-f4b2782a70d0","type":"FUNNEL","groupId":"62dd0a2b-528e-3db2-af01-43a69b2b4110","name":"Funnel","comments":"","instanceIdentifier":"6c6bead2-f6ba-363e-cf97-1615a827ee4a"},"labelIndex":0,"zIndex":2,"selectedRelationships":["failure"],"backPressureObjectThreshold":10000,"backPressureDataSizeThreshold":"1 GB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","partitioningAttribute":"","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"62dd0a2b-528e-3db2-af01-43a69b2b4110"}],"labels":[{"identifier":"75fc465e-0198-1000-3b14-99f546b0a46c","instanceIdentifier":"b28dec90-772b-3002-e966-ca723b6562ff","position":{"x":-248.0,"y":-496.0},"label":"Depending on the workflow you might need to change the ' Use Avro Logical Types' to false (careful as this can mess up json processing/parsing, usually we set it to false, the most troublesome fields are datetimes/timestamps.","zIndex":2,"width":256.0,"height":128.0,"style":{"font-size":"12px"},"componentType":"LABEL","groupIdentifier":"62dd0a2b-528e-3db2-af01-43a69b2b4110"},{"identifier":"75fd9c6f-0198-1000-b4f4-773365180dc6","instanceIdentifier":"fb5daa51-c311-3364-334d-7c42bc1a8e5b","position":{"x":-248.0,"y":-288.0},"label":"The 'Index' is the index name, takes the value of the ${generatetablefetch.tableName} which is a FlowFile property (not in the flowfile contents, but the property of such file).\n\nIMPORTANT:\n ID Record Path - this should be the name of the unique UUID of each of your records, defined as '/property_name' make note of the '/'. If you don't set this you might end up with duplicates as Elastic/Open(search) will generate its own uuids, obviously it wont care. An alternative is to generate a hash from the contents of the FlowFile and use it as an UUID.\n","zIndex":2,"width":624.0,"height":128.0,"style":{"font-size":"12px"},"componentType":"LABEL","groupIdentifier":"62dd0a2b-528e-3db2-af01-43a69b2b4110"},{"identifier":"5f0d9248-c0d1-3a12-9a11-f83682994ea6","instanceIdentifier":"f9fe445b-51a5-31fc-35a2-44fb8d5d59e2","position":{"x":-248.0,"y":-688.0},"label":"Please note the max-value column, it should be an integer and a primary key, for fast record retrieval.","zIndex":2,"width":256.0,"height":128.0,"style":{"font-size":"12px"},"componentType":"LABEL","groupIdentifier":"62dd0a2b-528e-3db2-af01-43a69b2b4110"},{"identifier":"df3a9fcc-95e1-38f8-a5de-d437067631ff","instanceIdentifier":"2feee655-8643-3a47-6ac7-84c4a1a4143b","position":{"x":-584.0,"y":-752.0},"label":"Simple ingestion from PgSQL to ElasticSearch","zIndex":1,"width":296.0,"height":24.0,"style":{"font-size":"12px"},"componentType":"LABEL","groupIdentifier":"62dd0a2b-528e-3db2-af01-43a69b2b4110"}],"funnels":[{"identifier":"a8b3cffd-32c7-33c3-bf00-f4b2782a70d0","instanceIdentifier":"6c6bead2-f6ba-363e-cf97-1615a827ee4a","position":{"x":-912.0,"y":-640.0},"componentType":"FUNNEL","groupIdentifier":"62dd0a2b-528e-3db2-af01-43a69b2b4110"},{"identifier":"518aca05-0197-1000-00c9-e6e151102f62","instanceIdentifier":"5bc9b53c-d3de-3ef5-6186-faee77d5f3cf","position":{"x":-912.0,"y":-248.0},"componentType":"FUNNEL","groupIdentifier":"62dd0a2b-528e-3db2-af01-43a69b2b4110"},{"identifier":"518a227f-0197-1000-c916-878366970718","instanceIdentifier":"932fe33f-692b-39b5-9fae-c69ac9c23fe9","position":{"x":-913.0,"y":-446.0},"componentType":"FUNNEL","groupIdentifier":"62dd0a2b-528e-3db2-af01-43a69b2b4110"},{"identifier":"518b162d-0197-1000-931b-72a60e7fcb87","instanceIdentifier":"331295b4-db6e-3fbc-ff90-eb89a70d717b","position":{"x":-912.0,"y":-120.0},"componentType":"FUNNEL","groupIdentifier":"62dd0a2b-528e-3db2-af01-43a69b2b4110"}],"controllerServices":[{"identifier":"384cff7c-0232-3e2c-bc72-37e5bdc266bc","instanceIdentifier":"ec2e51ac-b204-3780-383f-2720c7815e8e","name":"AvroReader","comments":"","type":"org.apache.nifi.avro.AvroReader","bundle":{"group":"org.apache.nifi","artifact":"nifi-record-serialization-services-nar","version":"2.10.0"},"properties":{"Schema Branch":null,"Schema Reference Reader":null,"Schema Text":"${avro.schema}","Schema Version":null,"Cache Size":"1000","Fast Reader Enabled":"true","Schema Registry":null,"Schema Access Strategy":"embedded-avro-schema","Schema Name":"${schema.name}"},"propertyDescriptors":{"Schema Branch":{"name":"Schema Branch","displayName":"Schema Branch","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Schema Reference Reader":{"name":"Schema Reference Reader","displayName":"Schema Reference Reader","identifiesControllerService":true,"sensitive":false,"dynamic":false},"Schema Text":{"name":"Schema Text","displayName":"Schema Text","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Schema Version":{"name":"Schema Version","displayName":"Schema Version","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Cache Size":{"name":"Cache Size","displayName":"Cache Size","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Fast Reader Enabled":{"name":"Fast Reader Enabled","displayName":"Fast Reader Enabled","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Schema Registry":{"name":"Schema Registry","displayName":"Schema Registry","identifiesControllerService":true,"sensitive":false,"dynamic":false},"Schema Access Strategy":{"name":"Schema Access Strategy","displayName":"Schema Access Strategy","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Schema Name":{"name":"Schema Name","displayName":"Schema Name","identifiesControllerService":false,"sensitive":false,"dynamic":false}},"controllerServiceApis":[{"type":"org.apache.nifi.serialization.RecordReaderFactory","bundle":{"group":"org.apache.nifi","artifact":"nifi-standard-services-api-nar","version":"2.10.0"}}],"scheduledState":"DISABLED","bulletinLevel":"WARN","componentType":"CONTROLLER_SERVICE","groupIdentifier":"62dd0a2b-528e-3db2-af01-43a69b2b4110"},{"identifier":"bb71c362-4fc9-314e-96ed-50c123266093","instanceIdentifier":"6d688e73-6eb9-3da3-c68a-f4b0e2c3ee4b","name":"JsonRecordSetWriter","comments":"","type":"org.apache.nifi.json.JsonRecordSetWriter","bundle":{"group":"org.apache.nifi","artifact":"nifi-record-serialization-services-nar","version":"2.10.0"},"properties":{"Schema Branch":null,"Schema Reference Reader":null,"Schema Text":"${avro.schema}","Allow Scientific Notation":"false","Compression Level":"1","Output Grouping":"output-array","Schema Registry":null,"Schema Access Strategy":"inherit-record-schema","Schema Name":"${schema.name}","Timestamp Format":null,"Date Format":null,"Pretty Print JSON":"false","Suppress Null Values":"never-suppress","Schema Write Strategy":"no-schema","Serialized JSON Input Handling":"ENABLED","Compression Format":"none","Schema Cache":null,"Schema Version":null,"Time Format":null,"Schema Reference Writer":null},"propertyDescriptors":{"Schema Branch":{"name":"Schema Branch","displayName":"Schema Branch","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Schema Reference Reader":{"name":"Schema Reference Reader","displayName":"Schema Reference Reader","identifiesControllerService":true,"sensitive":false,"dynamic":false},"Schema Text":{"name":"Schema Text","displayName":"Schema Text","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Allow Scientific Notation":{"name":"Allow Scientific Notation","displayName":"Allow Scientific Notation","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Compression Level":{"name":"Compression Level","displayName":"Compression Level","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Output Grouping":{"name":"Output Grouping","displayName":"Output Grouping","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Schema Registry":{"name":"Schema Registry","displayName":"Schema Registry","identifiesControllerService":true,"sensitive":false,"dynamic":false},"Schema Access Strategy":{"name":"Schema Access Strategy","displayName":"Schema Access Strategy","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Schema Name":{"name":"Schema Name","displayName":"Schema Name","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Timestamp Format":{"name":"Timestamp Format","displayName":"Timestamp Format","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Date Format":{"name":"Date Format","displayName":"Date Format","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Pretty Print JSON":{"name":"Pretty Print JSON","displayName":"Pretty Print JSON","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Suppress Null Values":{"name":"Suppress Null Values","displayName":"Suppress Null Values","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Schema Write Strategy":{"name":"Schema Write Strategy","displayName":"Schema Write Strategy","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Serialized JSON Input Handling":{"name":"Serialized JSON Input Handling","displayName":"Serialized JSON Input Handling","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Compression Format":{"name":"Compression Format","displayName":"Compression Format","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Schema Cache":{"name":"Schema Cache","displayName":"Schema Cache","identifiesControllerService":true,"sensitive":false,"dynamic":false},"Schema Version":{"name":"Schema Version","displayName":"Schema Version","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Time Format":{"name":"Time Format","displayName":"Time Format","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Schema Reference Writer":{"name":"Schema Reference Writer","displayName":"Schema Reference Writer","identifiesControllerService":true,"sensitive":false,"dynamic":false}},"controllerServiceApis":[{"type":"org.apache.nifi.serialization.RecordSetWriterFactory","bundle":{"group":"org.apache.nifi","artifact":"nifi-standard-services-api-nar","version":"2.10.0"}}],"scheduledState":"DISABLED","bulletinLevel":"WARN","componentType":"CONTROLLER_SERVICE","groupIdentifier":"62dd0a2b-528e-3db2-af01-43a69b2b4110"},{"identifier":"ae4d900f-d06e-3cc3-acc5-4f6f222a5064","instanceIdentifier":"2dc19211-4f9d-3ef5-17c3-57f5bf9c01c3","name":"DBCPConnectionPool","comments":"","type":"org.apache.nifi.dbcp.DBCPConnectionPool","bundle":{"group":"org.apache.nifi","artifact":"nifi-dbcp-service-nar","version":"2.10.0"},"properties":{"Maximum Idle Connections":"8","Database Driver Locations":"/opt/nifi/drivers/postgresql-42.7.7.jar","Max Wait Time":"500 millis","Database Driver Class Name":"org.postgresql.Driver","Minimum Idle Connections":"0","Kerberos User Service":null,"Max Total Connections":"8","Validation Query":null,"Database Password Provider":null,"Database Connection URL":"jdbc:postgresql://samples-db:5432/db_samples","Password Source":"PASSWORD","Time Between Eviction Runs":"-1","Database User":"test","Soft Minimum Evictable Idle Time":"-1","Maximum Connection Lifetime":"-1","Minimum Evictable Idle Time":"30 mins"},"propertyDescriptors":{"Maximum Idle Connections":{"name":"Maximum Idle Connections","displayName":"Maximum Idle Connections","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Database Driver Locations":{"name":"Database Driver Locations","displayName":"Database Driver Locations","identifiesControllerService":false,"sensitive":false,"dynamic":false,"resourceDefinition":{"cardinality":"MULTIPLE","resourceTypes":["DIRECTORY","FILE","URL"]}},"Max Wait Time":{"name":"Max Wait Time","displayName":"Max Wait Time","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Database Driver Class Name":{"name":"Database Driver Class Name","displayName":"Database Driver Class Name","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Minimum Idle Connections":{"name":"Minimum Idle Connections","displayName":"Minimum Idle Connections","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Kerberos User Service":{"name":"Kerberos User Service","displayName":"Kerberos User Service","identifiesControllerService":true,"sensitive":false,"dynamic":false},"Max Total Connections":{"name":"Max Total Connections","displayName":"Max Total Connections","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Validation Query":{"name":"Validation Query","displayName":"Validation Query","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Database Password Provider":{"name":"Database Password Provider","displayName":"Database Password Provider","identifiesControllerService":true,"sensitive":false,"dynamic":false},"Database Connection URL":{"name":"Database Connection URL","displayName":"Database Connection URL","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Password Source":{"name":"Password Source","displayName":"Password Source","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Time Between Eviction Runs":{"name":"Time Between Eviction Runs","displayName":"Time Between Eviction Runs","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Database User":{"name":"Database User","displayName":"Database User","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Soft Minimum Evictable Idle Time":{"name":"Soft Minimum Evictable Idle Time","displayName":"Soft Minimum Evictable Idle Time","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Maximum Connection Lifetime":{"name":"Maximum Connection Lifetime","displayName":"Maximum Connection Lifetime","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Minimum Evictable Idle Time":{"name":"Minimum Evictable Idle Time","displayName":"Minimum Evictable Idle Time","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Password":{"name":"Password","displayName":"Password","identifiesControllerService":false,"sensitive":true,"dynamic":false}},"controllerServiceApis":[{"type":"org.apache.nifi.dbcp.DBCPService","bundle":{"group":"org.apache.nifi","artifact":"nifi-standard-services-api-nar","version":"2.10.0"}}],"scheduledState":"DISABLED","bulletinLevel":"WARN","componentType":"CONTROLLER_SERVICE","groupIdentifier":"62dd0a2b-528e-3db2-af01-43a69b2b4110"},{"identifier":"06280792-dc54-3d7a-9d35-76ce834dcec3","instanceIdentifier":"98519849-937c-37d0-869e-e9bb6841df08","name":"ElasticSearchClientServiceImpl","comments":"","type":"org.apache.nifi.elasticsearch.ElasticSearchClientServiceImpl","bundle":{"group":"org.apache.nifi","artifact":"nifi-elasticsearch-client-service-nar","version":"2.10.0"},"properties":{"Sniffer Request Timeout":"1 sec","Connect Timeout":"5000","HTTP Hosts":"https://elasticsearch-1:9200","Path Prefix":null,"API Key ID":null,"Enable Compression":"false","Read Timeout":"60000","Sniff on Failure":"false","Sniffer Failure Delay":"1 min","OAuth2 Access Token Provider":null,"Suppress Null and Empty Values":"always-suppress","Node Selector":"ANY","Proxy Configuration Service":null,"Strict Deprecation":"false","Sniff Cluster Nodes":"false","SSL Context Service":"28f235b7-a6af-3e75-b7d5-6fbc32ecf77c","Send Meta Header":"true","Username":"admin","Authorization Scheme":"BASIC","Sniffer Interval":"5 mins","Run As User":null,"Character Set":"UTF-8"},"propertyDescriptors":{"Sniffer Request Timeout":{"name":"Sniffer Request Timeout","displayName":"Sniffer Request Timeout","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Connect Timeout":{"name":"Connect Timeout","displayName":"Connect Timeout","identifiesControllerService":false,"sensitive":false,"dynamic":false},"API Key ID":{"name":"API Key ID","displayName":"API Key ID","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Enable Compression":{"name":"Enable Compression","displayName":"Enable Compression","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Sniff on Failure":{"name":"Sniff on Failure","displayName":"Sniff on Failure","identifiesControllerService":false,"sensitive":false,"dynamic":false},"OAuth2 Access Token Provider":{"name":"OAuth2 Access Token Provider","displayName":"OAuth2 Access Token Provider","identifiesControllerService":true,"sensitive":false,"dynamic":false},"Node Selector":{"name":"Node Selector","displayName":"Node Selector","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Proxy Configuration Service":{"name":"Proxy Configuration Service","displayName":"Proxy Configuration Service","identifiesControllerService":true,"sensitive":false,"dynamic":false},"Strict Deprecation":{"name":"Strict Deprecation","displayName":"Strict Deprecation","identifiesControllerService":false,"sensitive":false,"dynamic":false},"JWT Shared Secret":{"name":"JWT Shared Secret","displayName":"JWT Shared Secret","identifiesControllerService":false,"sensitive":true,"dynamic":false},"API Key":{"name":"API Key","displayName":"API Key","identifiesControllerService":false,"sensitive":true,"dynamic":false},"Send Meta Header":{"name":"Send Meta Header","displayName":"Send Meta Header","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Authorization Scheme":{"name":"Authorization Scheme","displayName":"Authorization Scheme","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Password":{"name":"Password","displayName":"Password","identifiesControllerService":false,"sensitive":true,"dynamic":false},"HTTP Hosts":{"name":"HTTP Hosts","displayName":"HTTP Hosts","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Path Prefix":{"name":"Path Prefix","displayName":"Path Prefix","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Read Timeout":{"name":"Read Timeout","displayName":"Read Timeout","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Sniffer Failure Delay":{"name":"Sniffer Failure Delay","displayName":"Sniffer Failure Delay","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Suppress Null and Empty Values":{"name":"Suppress Null and Empty Values","displayName":"Suppress Null and Empty Values","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Sniff Cluster Nodes":{"name":"Sniff Cluster Nodes","displayName":"Sniff Cluster Nodes","identifiesControllerService":false,"sensitive":false,"dynamic":false},"SSL Context Service":{"name":"SSL Context Service","displayName":"SSL Context Service","identifiesControllerService":true,"sensitive":false,"dynamic":false},"Username":{"name":"Username","displayName":"Username","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Sniffer Interval":{"name":"Sniffer Interval","displayName":"Sniffer Interval","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Run As User":{"name":"Run As User","displayName":"Run As User","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Character Set":{"name":"Character Set","displayName":"Character Set","identifiesControllerService":false,"sensitive":false,"dynamic":false}},"controllerServiceApis":[{"type":"org.apache.nifi.elasticsearch.ElasticSearchClientService","bundle":{"group":"org.apache.nifi","artifact":"nifi-elasticsearch-client-service-api-nar","version":"2.10.0"}}],"scheduledState":"DISABLED","bulletinLevel":"WARN","componentType":"CONTROLLER_SERVICE","groupIdentifier":"62dd0a2b-528e-3db2-af01-43a69b2b4110"},{"identifier":"4ad0bf71-a5de-39f7-9f74-07213d115a9f","instanceIdentifier":"6332858d-825b-3517-5c9b-15f3e0578738","name":"AvroRecordSetWriter","comments":"","type":"org.apache.nifi.avro.AvroRecordSetWriter","bundle":{"group":"org.apache.nifi","artifact":"nifi-record-serialization-services-nar","version":"2.10.0"},"properties":{"Schema Branch":null,"Schema Reference Reader":null,"Schema Text":"${avro.schema}","Schema Registry":null,"Schema Access Strategy":"inherit-record-schema","Schema Name":"${schema.name}","Schema Write Strategy":"avro-embedded","Encoder Pool Size":"32","Compression Format":"NONE","Schema Cache":null,"Schema Version":null,"Cache Size":"1000","Schema Reference Writer":null},"propertyDescriptors":{"Schema Branch":{"name":"Schema Branch","displayName":"Schema Branch","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Schema Reference Reader":{"name":"Schema Reference Reader","displayName":"Schema Reference Reader","identifiesControllerService":true,"sensitive":false,"dynamic":false},"Schema Text":{"name":"Schema Text","displayName":"Schema Text","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Schema Registry":{"name":"Schema Registry","displayName":"Schema Registry","identifiesControllerService":true,"sensitive":false,"dynamic":false},"Schema Access Strategy":{"name":"Schema Access Strategy","displayName":"Schema Access Strategy","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Schema Name":{"name":"Schema Name","displayName":"Schema Name","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Schema Write Strategy":{"name":"Schema Write Strategy","displayName":"Schema Write Strategy","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Encoder Pool Size":{"name":"Encoder Pool Size","displayName":"Encoder Pool Size","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Compression Format":{"name":"Compression Format","displayName":"Compression Format","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Schema Cache":{"name":"Schema Cache","displayName":"Schema Cache","identifiesControllerService":true,"sensitive":false,"dynamic":false},"Schema Version":{"name":"Schema Version","displayName":"Schema Version","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Cache Size":{"name":"Cache Size","displayName":"Cache Size","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Schema Reference Writer":{"name":"Schema Reference Writer","displayName":"Schema Reference Writer","identifiesControllerService":true,"sensitive":false,"dynamic":false}},"controllerServiceApis":[{"type":"org.apache.nifi.serialization.RecordSetWriterFactory","bundle":{"group":"org.apache.nifi","artifact":"nifi-standard-services-api-nar","version":"2.10.0"}}],"scheduledState":"DISABLED","bulletinLevel":"WARN","componentType":"CONTROLLER_SERVICE","groupIdentifier":"62dd0a2b-528e-3db2-af01-43a69b2b4110"},{"identifier":"28f235b7-a6af-3e75-b7d5-6fbc32ecf77c","instanceIdentifier":"1d89800d-019f-1000-f19d-d807d7b00440","name":"StandardSSLContextService","comments":"","type":"org.apache.nifi.ssl.StandardSSLContextService","bundle":{"group":"org.apache.nifi","artifact":"nifi-ssl-context-service-nar","version":"2.10.0"},"properties":{"Truststore Type":"PKCS12","Keystore Type":"JKS","Truststore Filename":"/security/certificates/elastic/opensearch/elasticsearch/elasticsearch-1/elasticsearch-1-truststore.key","TLS Protocol":"TLS","Keystore Filename":"/security/certificates/elastic/opensearch/elasticsearch/elasticsearch-1/elasticsearch-1-keystore.jks"},"propertyDescriptors":{"Truststore Type":{"name":"Truststore Type","displayName":"Truststore Type","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Key Password":{"name":"Key Password","displayName":"Key Password","identifiesControllerService":false,"sensitive":true,"dynamic":false},"Keystore Type":{"name":"Keystore Type","displayName":"Keystore Type","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Truststore Filename":{"name":"Truststore Filename","displayName":"Truststore Filename","identifiesControllerService":false,"sensitive":false,"dynamic":false,"resourceDefinition":{"cardinality":"SINGLE","resourceTypes":["FILE"]}},"Keystore Password":{"name":"Keystore Password","displayName":"Keystore Password","identifiesControllerService":false,"sensitive":true,"dynamic":false},"Truststore Password":{"name":"Truststore Password","displayName":"Truststore Password","identifiesControllerService":false,"sensitive":true,"dynamic":false},"TLS Protocol":{"name":"TLS Protocol","displayName":"TLS Protocol","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Keystore Filename":{"name":"Keystore Filename","displayName":"Keystore Filename","identifiesControllerService":false,"sensitive":false,"dynamic":false,"resourceDefinition":{"cardinality":"SINGLE","resourceTypes":["FILE"]}}},"controllerServiceApis":[{"type":"org.apache.nifi.ssl.SSLContextService","bundle":{"group":"org.apache.nifi","artifact":"nifi-standard-services-api-nar","version":"2.10.0"}},{"type":"org.apache.nifi.ssl.SSLContextProvider","bundle":{"group":"org.apache.nifi","artifact":"nifi-standard-services-api-nar","version":"2.10.0"}}],"scheduledState":"DISABLED","bulletinLevel":"WARN","componentType":"CONTROLLER_SERVICE","groupIdentifier":"62dd0a2b-528e-3db2-af01-43a69b2b4110"}],"defaultFlowFileExpiration":"0 sec","defaultBackPressureObjectThreshold":10000,"defaultBackPressureDataSizeThreshold":"1 GB","scheduledState":"ENABLED","executionEngine":"INHERITED","maxConcurrentTasks":1,"statelessFlowTimeout":"1 min","flowFileConcurrency":"UNBOUNDED","flowFileOutboundPolicy":"STREAM_WHEN_AVAILABLE","componentType":"PROCESS_GROUP"},"externalControllerServices":{},"parameterContexts":{},"flowEncodingVersion":"1.0","parameterProviders":{},"latest":false} \ No newline at end of file +{"flowContents":{"identifier":"62dd0a2b-528e-3db2-af01-43a69b2b4110","instanceIdentifier":"1d75b764-019f-1000-b3ce-e679959f9af7","name":"opensearch_ingest_docs_db_to_es","comments":"","position":{"x":384.0,"y":347.0},"processGroups":[],"remoteProcessGroups":[],"processors":[{"identifier":"9496f814-758e-3ff4-be63-09cfd4e2a7b5","instanceIdentifier":"9a73d2f8-fae5-314f-a043-96aa082f7ab5","name":"ExecuteSQLRecord","comments":"","position":{"x":-608.0,"y":-496.0},"type":"org.apache.nifi.processors.standard.ExecuteSQLRecord","bundle":{"group":"org.apache.nifi","artifact":"nifi-standard-nar","version":"2.10.0"},"properties":{"Set Auto Commit":"true","Max Wait Time":"0 seconds","SQL Query":null,"Normalize Table/Column Names":"false","Database Connection Pooling Service":"ae4d900f-d06e-3cc3-acc5-4f6f222a5064","Default Decimal Scale":"0","SQL Post-Query":null,"Fetch Size":"0","Output Batch Size":"0","Use Avro Logical Types":"true","Record Writer":"4ad0bf71-a5de-39f7-9f74-07213d115a9f","SQL Pre-Query":null,"Default Decimal Precision":"10","Max Rows Per FlowFile":"0"},"propertyDescriptors":{"Set Auto Commit":{"name":"Set Auto Commit","displayName":"Set Auto Commit","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Max Wait Time":{"name":"Max Wait Time","displayName":"Max Wait Time","identifiesControllerService":false,"sensitive":false,"dynamic":false},"SQL Query":{"name":"SQL Query","displayName":"SQL Query","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Normalize Table/Column Names":{"name":"Normalize Table/Column Names","displayName":"Normalize Table/Column Names","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Database Connection Pooling Service":{"name":"Database Connection Pooling Service","displayName":"Database Connection Pooling Service","identifiesControllerService":true,"sensitive":false,"dynamic":false},"Default Decimal Scale":{"name":"Default Decimal Scale","displayName":"Default Decimal Scale","identifiesControllerService":false,"sensitive":false,"dynamic":false},"SQL Post-Query":{"name":"SQL Post-Query","displayName":"SQL Post-Query","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Fetch Size":{"name":"Fetch Size","displayName":"Fetch Size","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Output Batch Size":{"name":"Output Batch Size","displayName":"Output Batch Size","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Use Avro Logical Types":{"name":"Use Avro Logical Types","displayName":"Use Avro Logical Types","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Record Writer":{"name":"Record Writer","displayName":"Record Writer","identifiesControllerService":true,"sensitive":false,"dynamic":false},"SQL Pre-Query":{"name":"SQL Pre-Query","displayName":"SQL Pre-Query","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Default Decimal Precision":{"name":"Default Decimal Precision","displayName":"Default Decimal Precision","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Max Rows Per FlowFile":{"name":"Max Rows Per FlowFile","displayName":"Max Rows Per FlowFile","identifiesControllerService":false,"sensitive":false,"dynamic":false}},"style":{},"schedulingPeriod":"0 sec","schedulingStrategy":"TIMER_DRIVEN","executionNode":"ALL","penaltyDuration":"30 sec","yieldDuration":"1 sec","bulletinLevel":"WARN","runDurationMillis":0,"concurrentlySchedulableTaskCount":1,"autoTerminatedRelationships":[],"scheduledState":"ENABLED","retryCount":10,"retriedRelationships":[],"backoffMechanism":"PENALIZE_FLOWFILE","maxBackoffPeriod":"10 mins","componentType":"PROCESSOR","groupIdentifier":"62dd0a2b-528e-3db2-af01-43a69b2b4110"},{"identifier":"9c434759-d273-371e-a0dc-6e1f770d6a98","instanceIdentifier":"7cbd2dba-4532-3bb2-2ca0-cc42e36294af","name":"GenerateTableFetch-medical_reports_text","comments":"","position":{"x":-608.0,"y":-688.0},"type":"org.apache.nifi.processors.standard.GenerateTableFetch","bundle":{"group":"org.apache.nifi","artifact":"nifi-standard-nar","version":"2.10.0"},"properties":{"Table Name":"medical_reports_text","Max Wait Time":"0 seconds","Partition Size":"10000","Database Connection Pooling Service":"ae4d900f-d06e-3cc3-acc5-4f6f222a5064","Database Type":"PostgreSQL","Columns to Return":"sampleid,typeid,dct,filename,\"document\",CAST(docid AS VARCHAR(255)) AS docid","Maximum-value Columns":"docid","Custom ORDER BY Column":null,"Output Empty FlowFile on Zero Results":"false","Database Dialect Service":null,"Column for Value Partitioning":"docid","Additional WHERE Clause":null},"propertyDescriptors":{"Table Name":{"name":"Table Name","displayName":"Table Name","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Max Wait Time":{"name":"Max Wait Time","displayName":"Max Wait Time","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Partition Size":{"name":"Partition Size","displayName":"Partition Size","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Database Connection Pooling Service":{"name":"Database Connection Pooling Service","displayName":"Database Connection Pooling Service","identifiesControllerService":true,"sensitive":false,"dynamic":false},"Database Type":{"name":"Database Type","displayName":"Database Type","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Columns to Return":{"name":"Columns to Return","displayName":"Columns to Return","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Maximum-value Columns":{"name":"Maximum-value Columns","displayName":"Maximum-value Columns","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Custom ORDER BY Column":{"name":"Custom ORDER BY Column","displayName":"Custom ORDER BY Column","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Output Empty FlowFile on Zero Results":{"name":"Output Empty FlowFile on Zero Results","displayName":"Output Empty FlowFile on Zero Results","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Database Dialect Service":{"name":"Database Dialect Service","displayName":"Database Dialect Service","identifiesControllerService":true,"sensitive":false,"dynamic":false},"Column for Value Partitioning":{"name":"Column for Value Partitioning","displayName":"Column for Value Partitioning","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Additional WHERE Clause":{"name":"Additional WHERE Clause","displayName":"Additional WHERE Clause","identifiesControllerService":false,"sensitive":false,"dynamic":false}},"style":{},"schedulingPeriod":"0 sec","schedulingStrategy":"TIMER_DRIVEN","executionNode":"ALL","penaltyDuration":"30 sec","yieldDuration":"1 sec","bulletinLevel":"WARN","runDurationMillis":0,"concurrentlySchedulableTaskCount":1,"autoTerminatedRelationships":[],"scheduledState":"ENABLED","retryCount":10,"retriedRelationships":[],"backoffMechanism":"PENALIZE_FLOWFILE","maxBackoffPeriod":"10 mins","componentType":"PROCESSOR","groupIdentifier":"62dd0a2b-528e-3db2-af01-43a69b2b4110"},{"identifier":"ae73dcb2-f24b-3e7a-911d-a7b3c7e0678e","instanceIdentifier":"40a6e665-bf5d-346a-ea03-ef647cd8f90f","name":"PutElasticsearchRecord","comments":"","position":{"x":-608.0,"y":-288.0},"type":"org.apache.nifi.processors.elasticsearch.PutElasticsearchRecord","bundle":{"group":"org.apache.nifi","artifact":"nifi-elasticsearch-restapi-nar","version":"2.10.0"},"properties":{"Timestamp Value":null,"Max JSON Field String Length":"64 MB","Dynamic Templates Record Path":null,"Index Operation":"index","Index":"${generatetablefetch.tableName}","Record Reader":"384cff7c-0232-3e2c-bc72-37e5bdc266bc","Date Format":null,"Retain Record Path ID Field":"false","Time Format":null,"Result Record Writer":"bb71c362-4fc9-314e-96ed-50c123266093","Treat Not Found as Success":"true","Group Results by Bulk Error Type":"false","Scripted Upsert Record Path":null,"Index Operation Record Path":null,"Type Record Path":null,"ID Record Path":"/docid","Log Error Responses":"false","Script Record Path":null,"Output Error Responses":"false","Retain Record Timestamp":"false","Timestamp Format":null,"Type":null,"Batch Size":"10000","Client Service":"06280792-dc54-3d7a-9d35-76ce834dcec3","Index Record Path":null,"Timestamp Record Path":null},"propertyDescriptors":{"Timestamp Value":{"name":"Timestamp Value","displayName":"Timestamp Value","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Max JSON Field String Length":{"name":"Max JSON Field String Length","displayName":"Max JSON Field String Length","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Dynamic Templates Record Path":{"name":"Dynamic Templates Record Path","displayName":"Dynamic Templates Record Path","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Index Operation":{"name":"Index Operation","displayName":"Index Operation","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Index":{"name":"Index","displayName":"Index","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Record Reader":{"name":"Record Reader","displayName":"Record Reader","identifiesControllerService":true,"sensitive":false,"dynamic":false},"Date Format":{"name":"Date Format","displayName":"Date Format","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Retain Record Path ID Field":{"name":"Retain Record Path ID Field","displayName":"Retain Record Path ID Field","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Time Format":{"name":"Time Format","displayName":"Time Format","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Result Record Writer":{"name":"Result Record Writer","displayName":"Result Record Writer","identifiesControllerService":true,"sensitive":false,"dynamic":false},"Treat Not Found as Success":{"name":"Treat Not Found as Success","displayName":"Treat Not Found as Success","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Group Results by Bulk Error Type":{"name":"Group Results by Bulk Error Type","displayName":"Group Results by Bulk Error Type","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Scripted Upsert Record Path":{"name":"Scripted Upsert Record Path","displayName":"Scripted Upsert Record Path","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Index Operation Record Path":{"name":"Index Operation Record Path","displayName":"Index Operation Record Path","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Type Record Path":{"name":"Type Record Path","displayName":"Type Record Path","identifiesControllerService":false,"sensitive":false,"dynamic":false},"ID Record Path":{"name":"ID Record Path","displayName":"ID Record Path","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Log Error Responses":{"name":"Log Error Responses","displayName":"Log Error Responses","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Script Record Path":{"name":"Script Record Path","displayName":"Script Record Path","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Output Error Responses":{"name":"Output Error Responses","displayName":"Output Error Responses","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Retain Record Timestamp":{"name":"Retain Record Timestamp","displayName":"Retain Record Timestamp","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Timestamp Format":{"name":"Timestamp Format","displayName":"Timestamp Format","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Type":{"name":"Type","displayName":"Type","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Batch Size":{"name":"Batch Size","displayName":"Batch Size","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Client Service":{"name":"Client Service","displayName":"Client Service","identifiesControllerService":true,"sensitive":false,"dynamic":false},"Index Record Path":{"name":"Index Record Path","displayName":"Index Record Path","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Timestamp Record Path":{"name":"Timestamp Record Path","displayName":"Timestamp Record Path","identifiesControllerService":false,"sensitive":false,"dynamic":false}},"style":{},"schedulingPeriod":"0 sec","schedulingStrategy":"TIMER_DRIVEN","executionNode":"ALL","penaltyDuration":"30 sec","yieldDuration":"1 sec","bulletinLevel":"WARN","runDurationMillis":0,"concurrentlySchedulableTaskCount":1,"autoTerminatedRelationships":["original","successful"],"scheduledState":"ENABLED","retryCount":10,"retriedRelationships":[],"backoffMechanism":"PENALIZE_FLOWFILE","maxBackoffPeriod":"10 mins","componentType":"PROCESSOR","groupIdentifier":"62dd0a2b-528e-3db2-af01-43a69b2b4110"}],"inputPorts":[],"outputPorts":[],"connections":[{"identifier":"992d8825-b10c-3844-9ba5-5076f20673b7","instanceIdentifier":"91e284b1-60de-3e49-0424-a24a3f839e97","name":"","source":{"id":"9c434759-d273-371e-a0dc-6e1f770d6a98","type":"PROCESSOR","groupId":"62dd0a2b-528e-3db2-af01-43a69b2b4110","name":"GenerateTableFetch-medical_reports_text","comments":"","instanceIdentifier":"7cbd2dba-4532-3bb2-2ca0-cc42e36294af"},"destination":{"id":"9496f814-758e-3ff4-be63-09cfd4e2a7b5","type":"PROCESSOR","groupId":"62dd0a2b-528e-3db2-af01-43a69b2b4110","name":"ExecuteSQLRecord","comments":"","instanceIdentifier":"9a73d2f8-fae5-314f-a043-96aa082f7ab5"},"labelIndex":0,"zIndex":1,"selectedRelationships":["success"],"backPressureObjectThreshold":10000,"backPressureDataSizeThreshold":"1 GB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","partitioningAttribute":"","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"62dd0a2b-528e-3db2-af01-43a69b2b4110"},{"identifier":"4daeaa34-b854-324a-b970-4878b7095d0f","instanceIdentifier":"64b44b06-87c1-3aaf-00e6-43241d85aad8","name":"","source":{"id":"9496f814-758e-3ff4-be63-09cfd4e2a7b5","type":"PROCESSOR","groupId":"62dd0a2b-528e-3db2-af01-43a69b2b4110","name":"ExecuteSQLRecord","comments":"","instanceIdentifier":"9a73d2f8-fae5-314f-a043-96aa082f7ab5"},"destination":{"id":"518a227f-0197-1000-c916-878366970718","type":"FUNNEL","groupId":"62dd0a2b-528e-3db2-af01-43a69b2b4110","name":"Funnel","comments":"","instanceIdentifier":"932fe33f-692b-39b5-9fae-c69ac9c23fe9"},"labelIndex":0,"zIndex":3,"selectedRelationships":["failure"],"backPressureObjectThreshold":10000,"backPressureDataSizeThreshold":"1 GB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","partitioningAttribute":"","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"62dd0a2b-528e-3db2-af01-43a69b2b4110"},{"identifier":"bf5a6660-9dd9-3efc-bdb3-fab678070664","instanceIdentifier":"60223890-f9c7-3291-6cd6-9ae237099fe9","name":"","source":{"id":"9496f814-758e-3ff4-be63-09cfd4e2a7b5","type":"PROCESSOR","groupId":"62dd0a2b-528e-3db2-af01-43a69b2b4110","name":"ExecuteSQLRecord","comments":"","instanceIdentifier":"9a73d2f8-fae5-314f-a043-96aa082f7ab5"},"destination":{"id":"ae73dcb2-f24b-3e7a-911d-a7b3c7e0678e","type":"PROCESSOR","groupId":"62dd0a2b-528e-3db2-af01-43a69b2b4110","name":"PutElasticsearchRecord","comments":"","instanceIdentifier":"40a6e665-bf5d-346a-ea03-ef647cd8f90f"},"labelIndex":0,"zIndex":4,"selectedRelationships":["success"],"backPressureObjectThreshold":10000,"backPressureDataSizeThreshold":"1 GB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","partitioningAttribute":"","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"62dd0a2b-528e-3db2-af01-43a69b2b4110"},{"identifier":"d697dfd0-3e44-385a-a55c-5b52403c9903","instanceIdentifier":"6508ef68-6f8d-38ca-aaa7-21521e910031","name":"","source":{"id":"ae73dcb2-f24b-3e7a-911d-a7b3c7e0678e","type":"PROCESSOR","groupId":"62dd0a2b-528e-3db2-af01-43a69b2b4110","name":"PutElasticsearchRecord","comments":"","instanceIdentifier":"40a6e665-bf5d-346a-ea03-ef647cd8f90f"},"destination":{"id":"518b162d-0197-1000-931b-72a60e7fcb87","type":"FUNNEL","groupId":"62dd0a2b-528e-3db2-af01-43a69b2b4110","name":"Funnel","comments":"","instanceIdentifier":"331295b4-db6e-3fbc-ff90-eb89a70d717b"},"labelIndex":0,"zIndex":6,"selectedRelationships":["retry"],"backPressureObjectThreshold":10000,"backPressureDataSizeThreshold":"1 GB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","partitioningAttribute":"","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"62dd0a2b-528e-3db2-af01-43a69b2b4110"},{"identifier":"f24c9db6-764c-3075-a921-8f5a3b5a1d29","instanceIdentifier":"75e50475-b018-32a4-a3ae-b6e571b70855","name":"","source":{"id":"ae73dcb2-f24b-3e7a-911d-a7b3c7e0678e","type":"PROCESSOR","groupId":"62dd0a2b-528e-3db2-af01-43a69b2b4110","name":"PutElasticsearchRecord","comments":"","instanceIdentifier":"40a6e665-bf5d-346a-ea03-ef647cd8f90f"},"destination":{"id":"518aca05-0197-1000-00c9-e6e151102f62","type":"FUNNEL","groupId":"62dd0a2b-528e-3db2-af01-43a69b2b4110","name":"Funnel","comments":"","instanceIdentifier":"5bc9b53c-d3de-3ef5-6186-faee77d5f3cf"},"labelIndex":0,"zIndex":5,"selectedRelationships":["failure","errors"],"backPressureObjectThreshold":10000,"backPressureDataSizeThreshold":"1 GB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","partitioningAttribute":"","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"62dd0a2b-528e-3db2-af01-43a69b2b4110"},{"identifier":"ba753c99-d884-3eef-8618-8fd11a022a32","instanceIdentifier":"6afb924d-728a-3d11-4746-e6ebae4a896c","name":"","source":{"id":"9c434759-d273-371e-a0dc-6e1f770d6a98","type":"PROCESSOR","groupId":"62dd0a2b-528e-3db2-af01-43a69b2b4110","name":"GenerateTableFetch-medical_reports_text","comments":"","instanceIdentifier":"7cbd2dba-4532-3bb2-2ca0-cc42e36294af"},"destination":{"id":"a8b3cffd-32c7-33c3-bf00-f4b2782a70d0","type":"FUNNEL","groupId":"62dd0a2b-528e-3db2-af01-43a69b2b4110","name":"Funnel","comments":"","instanceIdentifier":"6c6bead2-f6ba-363e-cf97-1615a827ee4a"},"labelIndex":0,"zIndex":2,"selectedRelationships":["failure"],"backPressureObjectThreshold":10000,"backPressureDataSizeThreshold":"1 GB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","partitioningAttribute":"","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"62dd0a2b-528e-3db2-af01-43a69b2b4110"}],"labels":[{"identifier":"75fc465e-0198-1000-3b14-99f546b0a46c","instanceIdentifier":"b28dec90-772b-3002-e966-ca723b6562ff","position":{"x":-248.0,"y":-496.0},"label":"Depending on the workflow you might need to change the ' Use Avro Logical Types' to false (careful as this can mess up json processing/parsing, usually we set it to false, the most troublesome fields are datetimes/timestamps.","zIndex":2,"width":256.0,"height":128.0,"style":{"font-size":"12px"},"componentType":"LABEL","groupIdentifier":"62dd0a2b-528e-3db2-af01-43a69b2b4110"},{"identifier":"75fd9c6f-0198-1000-b4f4-773365180dc6","instanceIdentifier":"fb5daa51-c311-3364-334d-7c42bc1a8e5b","position":{"x":-248.0,"y":-288.0},"label":"The 'Index' is the index name, takes the value of the ${generatetablefetch.tableName} which is a FlowFile property (not in the flowfile contents, but the property of such file).\n\nIMPORTANT:\n ID Record Path - this should be the name of the unique UUID of each of your records, defined as '/property_name' make note of the '/'. If you don't set this you might end up with duplicates as Elastic/Open(search) will generate its own uuids, obviously it wont care. An alternative is to generate a hash from the contents of the FlowFile and use it as an UUID.\n","zIndex":2,"width":624.0,"height":128.0,"style":{"font-size":"12px"},"componentType":"LABEL","groupIdentifier":"62dd0a2b-528e-3db2-af01-43a69b2b4110"},{"identifier":"5f0d9248-c0d1-3a12-9a11-f83682994ea6","instanceIdentifier":"f9fe445b-51a5-31fc-35a2-44fb8d5d59e2","position":{"x":-248.0,"y":-688.0},"label":"Please note the max-value column, it should be an integer and a primary key, for fast record retrieval.","zIndex":2,"width":256.0,"height":128.0,"style":{"font-size":"12px"},"componentType":"LABEL","groupIdentifier":"62dd0a2b-528e-3db2-af01-43a69b2b4110"},{"identifier":"df3a9fcc-95e1-38f8-a5de-d437067631ff","instanceIdentifier":"2feee655-8643-3a47-6ac7-84c4a1a4143b","position":{"x":-584.0,"y":-752.0},"label":"Simple ingestion from PgSQL to ElasticSearch","zIndex":1,"width":296.0,"height":24.0,"style":{"font-size":"12px"},"componentType":"LABEL","groupIdentifier":"62dd0a2b-528e-3db2-af01-43a69b2b4110"}],"funnels":[{"identifier":"a8b3cffd-32c7-33c3-bf00-f4b2782a70d0","instanceIdentifier":"6c6bead2-f6ba-363e-cf97-1615a827ee4a","position":{"x":-912.0,"y":-640.0},"componentType":"FUNNEL","groupIdentifier":"62dd0a2b-528e-3db2-af01-43a69b2b4110"},{"identifier":"518aca05-0197-1000-00c9-e6e151102f62","instanceIdentifier":"5bc9b53c-d3de-3ef5-6186-faee77d5f3cf","position":{"x":-912.0,"y":-248.0},"componentType":"FUNNEL","groupIdentifier":"62dd0a2b-528e-3db2-af01-43a69b2b4110"},{"identifier":"518a227f-0197-1000-c916-878366970718","instanceIdentifier":"932fe33f-692b-39b5-9fae-c69ac9c23fe9","position":{"x":-913.0,"y":-446.0},"componentType":"FUNNEL","groupIdentifier":"62dd0a2b-528e-3db2-af01-43a69b2b4110"},{"identifier":"518b162d-0197-1000-931b-72a60e7fcb87","instanceIdentifier":"331295b4-db6e-3fbc-ff90-eb89a70d717b","position":{"x":-912.0,"y":-120.0},"componentType":"FUNNEL","groupIdentifier":"62dd0a2b-528e-3db2-af01-43a69b2b4110"}],"controllerServices":[{"identifier":"384cff7c-0232-3e2c-bc72-37e5bdc266bc","instanceIdentifier":"ec2e51ac-b204-3780-383f-2720c7815e8e","name":"AvroReader","comments":"","type":"org.apache.nifi.avro.AvroReader","bundle":{"group":"org.apache.nifi","artifact":"nifi-record-serialization-services-nar","version":"2.10.0"},"properties":{"Schema Branch":null,"Schema Reference Reader":null,"Schema Text":"${avro.schema}","Schema Version":null,"Cache Size":"1000","Fast Reader Enabled":"true","Schema Registry":null,"Schema Access Strategy":"embedded-avro-schema","Schema Name":"${schema.name}"},"propertyDescriptors":{"Schema Branch":{"name":"Schema Branch","displayName":"Schema Branch","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Schema Reference Reader":{"name":"Schema Reference Reader","displayName":"Schema Reference Reader","identifiesControllerService":true,"sensitive":false,"dynamic":false},"Schema Text":{"name":"Schema Text","displayName":"Schema Text","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Schema Version":{"name":"Schema Version","displayName":"Schema Version","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Cache Size":{"name":"Cache Size","displayName":"Cache Size","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Fast Reader Enabled":{"name":"Fast Reader Enabled","displayName":"Fast Reader Enabled","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Schema Registry":{"name":"Schema Registry","displayName":"Schema Registry","identifiesControllerService":true,"sensitive":false,"dynamic":false},"Schema Access Strategy":{"name":"Schema Access Strategy","displayName":"Schema Access Strategy","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Schema Name":{"name":"Schema Name","displayName":"Schema Name","identifiesControllerService":false,"sensitive":false,"dynamic":false}},"controllerServiceApis":[{"type":"org.apache.nifi.serialization.RecordReaderFactory","bundle":{"group":"org.apache.nifi","artifact":"nifi-standard-services-api-nar","version":"2.10.0"}}],"scheduledState":"DISABLED","bulletinLevel":"WARN","componentType":"CONTROLLER_SERVICE","groupIdentifier":"62dd0a2b-528e-3db2-af01-43a69b2b4110"},{"identifier":"bb71c362-4fc9-314e-96ed-50c123266093","instanceIdentifier":"6d688e73-6eb9-3da3-c68a-f4b0e2c3ee4b","name":"JsonRecordSetWriter","comments":"","type":"org.apache.nifi.json.JsonRecordSetWriter","bundle":{"group":"org.apache.nifi","artifact":"nifi-record-serialization-services-nar","version":"2.10.0"},"properties":{"Schema Branch":null,"Schema Reference Reader":null,"Schema Text":"${avro.schema}","Allow Scientific Notation":"false","Compression Level":"1","Output Grouping":"output-array","Schema Registry":null,"Schema Access Strategy":"inherit-record-schema","Schema Name":"${schema.name}","Timestamp Format":null,"Date Format":null,"Pretty Print JSON":"false","Suppress Null Values":"never-suppress","Schema Write Strategy":"no-schema","Serialized JSON Input Handling":"ENABLED","Compression Format":"none","Schema Cache":null,"Schema Version":null,"Time Format":null,"Schema Reference Writer":null},"propertyDescriptors":{"Schema Branch":{"name":"Schema Branch","displayName":"Schema Branch","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Schema Reference Reader":{"name":"Schema Reference Reader","displayName":"Schema Reference Reader","identifiesControllerService":true,"sensitive":false,"dynamic":false},"Schema Text":{"name":"Schema Text","displayName":"Schema Text","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Allow Scientific Notation":{"name":"Allow Scientific Notation","displayName":"Allow Scientific Notation","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Compression Level":{"name":"Compression Level","displayName":"Compression Level","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Output Grouping":{"name":"Output Grouping","displayName":"Output Grouping","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Schema Registry":{"name":"Schema Registry","displayName":"Schema Registry","identifiesControllerService":true,"sensitive":false,"dynamic":false},"Schema Access Strategy":{"name":"Schema Access Strategy","displayName":"Schema Access Strategy","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Schema Name":{"name":"Schema Name","displayName":"Schema Name","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Timestamp Format":{"name":"Timestamp Format","displayName":"Timestamp Format","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Date Format":{"name":"Date Format","displayName":"Date Format","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Pretty Print JSON":{"name":"Pretty Print JSON","displayName":"Pretty Print JSON","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Suppress Null Values":{"name":"Suppress Null Values","displayName":"Suppress Null Values","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Schema Write Strategy":{"name":"Schema Write Strategy","displayName":"Schema Write Strategy","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Serialized JSON Input Handling":{"name":"Serialized JSON Input Handling","displayName":"Serialized JSON Input Handling","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Compression Format":{"name":"Compression Format","displayName":"Compression Format","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Schema Cache":{"name":"Schema Cache","displayName":"Schema Cache","identifiesControllerService":true,"sensitive":false,"dynamic":false},"Schema Version":{"name":"Schema Version","displayName":"Schema Version","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Time Format":{"name":"Time Format","displayName":"Time Format","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Schema Reference Writer":{"name":"Schema Reference Writer","displayName":"Schema Reference Writer","identifiesControllerService":true,"sensitive":false,"dynamic":false}},"controllerServiceApis":[{"type":"org.apache.nifi.serialization.RecordSetWriterFactory","bundle":{"group":"org.apache.nifi","artifact":"nifi-standard-services-api-nar","version":"2.10.0"}}],"scheduledState":"DISABLED","bulletinLevel":"WARN","componentType":"CONTROLLER_SERVICE","groupIdentifier":"62dd0a2b-528e-3db2-af01-43a69b2b4110"},{"identifier":"ae4d900f-d06e-3cc3-acc5-4f6f222a5064","instanceIdentifier":"2dc19211-4f9d-3ef5-17c3-57f5bf9c01c3","name":"DBCPConnectionPool","comments":"","type":"org.apache.nifi.dbcp.DBCPConnectionPool","bundle":{"group":"org.apache.nifi","artifact":"nifi-dbcp-service-nar","version":"2.10.0"},"properties":{"Maximum Idle Connections":"8","Database Driver Locations":"/opt/nifi/drivers/postgresql-42.7.7.jar","Max Wait Time":"500 millis","Database Driver Class Name":"org.postgresql.Driver","Minimum Idle Connections":"0","Kerberos User Service":null,"Max Total Connections":"8","Validation Query":null,"Database Password Provider":null,"Database Connection URL":"${POSTGRES_SAMPLES_JDBC_URL}","Password Source":"PASSWORD","Time Between Eviction Runs":"-1","Database User":"${POSTGRES_INGEST_USER_SAMPLES}","Password":"${POSTGRES_INGEST_PASSWORD_SAMPLES}","Soft Minimum Evictable Idle Time":"-1","Maximum Connection Lifetime":"-1","Minimum Evictable Idle Time":"30 mins"},"propertyDescriptors":{"Maximum Idle Connections":{"name":"Maximum Idle Connections","displayName":"Maximum Idle Connections","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Database Driver Locations":{"name":"Database Driver Locations","displayName":"Database Driver Locations","identifiesControllerService":false,"sensitive":false,"dynamic":false,"resourceDefinition":{"cardinality":"MULTIPLE","resourceTypes":["DIRECTORY","FILE","URL"]}},"Max Wait Time":{"name":"Max Wait Time","displayName":"Max Wait Time","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Database Driver Class Name":{"name":"Database Driver Class Name","displayName":"Database Driver Class Name","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Minimum Idle Connections":{"name":"Minimum Idle Connections","displayName":"Minimum Idle Connections","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Kerberos User Service":{"name":"Kerberos User Service","displayName":"Kerberos User Service","identifiesControllerService":true,"sensitive":false,"dynamic":false},"Max Total Connections":{"name":"Max Total Connections","displayName":"Max Total Connections","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Validation Query":{"name":"Validation Query","displayName":"Validation Query","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Database Password Provider":{"name":"Database Password Provider","displayName":"Database Password Provider","identifiesControllerService":true,"sensitive":false,"dynamic":false},"Database Connection URL":{"name":"Database Connection URL","displayName":"Database Connection URL","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Password Source":{"name":"Password Source","displayName":"Password Source","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Time Between Eviction Runs":{"name":"Time Between Eviction Runs","displayName":"Time Between Eviction Runs","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Database User":{"name":"Database User","displayName":"Database User","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Soft Minimum Evictable Idle Time":{"name":"Soft Minimum Evictable Idle Time","displayName":"Soft Minimum Evictable Idle Time","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Maximum Connection Lifetime":{"name":"Maximum Connection Lifetime","displayName":"Maximum Connection Lifetime","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Minimum Evictable Idle Time":{"name":"Minimum Evictable Idle Time","displayName":"Minimum Evictable Idle Time","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Password":{"name":"Password","displayName":"Password","identifiesControllerService":false,"sensitive":true,"dynamic":false}},"controllerServiceApis":[{"type":"org.apache.nifi.dbcp.DBCPService","bundle":{"group":"org.apache.nifi","artifact":"nifi-standard-services-api-nar","version":"2.10.0"}}],"scheduledState":"DISABLED","bulletinLevel":"WARN","componentType":"CONTROLLER_SERVICE","groupIdentifier":"62dd0a2b-528e-3db2-af01-43a69b2b4110"},{"identifier":"06280792-dc54-3d7a-9d35-76ce834dcec3","instanceIdentifier":"98519849-937c-37d0-869e-e9bb6841df08","name":"ElasticSearchClientServiceImpl","comments":"","type":"org.apache.nifi.elasticsearch.ElasticSearchClientServiceImpl","bundle":{"group":"org.apache.nifi","artifact":"nifi-elasticsearch-client-service-nar","version":"2.10.0"},"properties":{"Sniffer Request Timeout":"1 sec","Connect Timeout":"5000","HTTP Hosts":"https://elasticsearch-1:9200","Path Prefix":null,"API Key ID":null,"Enable Compression":"false","Read Timeout":"60000","Sniff on Failure":"false","Sniffer Failure Delay":"1 min","OAuth2 Access Token Provider":null,"Suppress Null and Empty Values":"always-suppress","Node Selector":"ANY","Proxy Configuration Service":null,"Strict Deprecation":"false","Sniff Cluster Nodes":"false","SSL Context Service":"28f235b7-a6af-3e75-b7d5-6fbc32ecf77c","Send Meta Header":"true","Username":"admin","Authorization Scheme":"BASIC","Sniffer Interval":"5 mins","Run As User":null,"Character Set":"UTF-8"},"propertyDescriptors":{"Sniffer Request Timeout":{"name":"Sniffer Request Timeout","displayName":"Sniffer Request Timeout","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Connect Timeout":{"name":"Connect Timeout","displayName":"Connect Timeout","identifiesControllerService":false,"sensitive":false,"dynamic":false},"API Key ID":{"name":"API Key ID","displayName":"API Key ID","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Enable Compression":{"name":"Enable Compression","displayName":"Enable Compression","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Sniff on Failure":{"name":"Sniff on Failure","displayName":"Sniff on Failure","identifiesControllerService":false,"sensitive":false,"dynamic":false},"OAuth2 Access Token Provider":{"name":"OAuth2 Access Token Provider","displayName":"OAuth2 Access Token Provider","identifiesControllerService":true,"sensitive":false,"dynamic":false},"Node Selector":{"name":"Node Selector","displayName":"Node Selector","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Proxy Configuration Service":{"name":"Proxy Configuration Service","displayName":"Proxy Configuration Service","identifiesControllerService":true,"sensitive":false,"dynamic":false},"Strict Deprecation":{"name":"Strict Deprecation","displayName":"Strict Deprecation","identifiesControllerService":false,"sensitive":false,"dynamic":false},"JWT Shared Secret":{"name":"JWT Shared Secret","displayName":"JWT Shared Secret","identifiesControllerService":false,"sensitive":true,"dynamic":false},"API Key":{"name":"API Key","displayName":"API Key","identifiesControllerService":false,"sensitive":true,"dynamic":false},"Send Meta Header":{"name":"Send Meta Header","displayName":"Send Meta Header","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Authorization Scheme":{"name":"Authorization Scheme","displayName":"Authorization Scheme","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Password":{"name":"Password","displayName":"Password","identifiesControllerService":false,"sensitive":true,"dynamic":false},"HTTP Hosts":{"name":"HTTP Hosts","displayName":"HTTP Hosts","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Path Prefix":{"name":"Path Prefix","displayName":"Path Prefix","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Read Timeout":{"name":"Read Timeout","displayName":"Read Timeout","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Sniffer Failure Delay":{"name":"Sniffer Failure Delay","displayName":"Sniffer Failure Delay","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Suppress Null and Empty Values":{"name":"Suppress Null and Empty Values","displayName":"Suppress Null and Empty Values","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Sniff Cluster Nodes":{"name":"Sniff Cluster Nodes","displayName":"Sniff Cluster Nodes","identifiesControllerService":false,"sensitive":false,"dynamic":false},"SSL Context Service":{"name":"SSL Context Service","displayName":"SSL Context Service","identifiesControllerService":true,"sensitive":false,"dynamic":false},"Username":{"name":"Username","displayName":"Username","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Sniffer Interval":{"name":"Sniffer Interval","displayName":"Sniffer Interval","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Run As User":{"name":"Run As User","displayName":"Run As User","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Character Set":{"name":"Character Set","displayName":"Character Set","identifiesControllerService":false,"sensitive":false,"dynamic":false}},"controllerServiceApis":[{"type":"org.apache.nifi.elasticsearch.ElasticSearchClientService","bundle":{"group":"org.apache.nifi","artifact":"nifi-elasticsearch-client-service-api-nar","version":"2.10.0"}}],"scheduledState":"DISABLED","bulletinLevel":"WARN","componentType":"CONTROLLER_SERVICE","groupIdentifier":"62dd0a2b-528e-3db2-af01-43a69b2b4110"},{"identifier":"4ad0bf71-a5de-39f7-9f74-07213d115a9f","instanceIdentifier":"6332858d-825b-3517-5c9b-15f3e0578738","name":"AvroRecordSetWriter","comments":"","type":"org.apache.nifi.avro.AvroRecordSetWriter","bundle":{"group":"org.apache.nifi","artifact":"nifi-record-serialization-services-nar","version":"2.10.0"},"properties":{"Schema Branch":null,"Schema Reference Reader":null,"Schema Text":"${avro.schema}","Schema Registry":null,"Schema Access Strategy":"inherit-record-schema","Schema Name":"${schema.name}","Schema Write Strategy":"avro-embedded","Encoder Pool Size":"32","Compression Format":"NONE","Schema Cache":null,"Schema Version":null,"Cache Size":"1000","Schema Reference Writer":null},"propertyDescriptors":{"Schema Branch":{"name":"Schema Branch","displayName":"Schema Branch","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Schema Reference Reader":{"name":"Schema Reference Reader","displayName":"Schema Reference Reader","identifiesControllerService":true,"sensitive":false,"dynamic":false},"Schema Text":{"name":"Schema Text","displayName":"Schema Text","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Schema Registry":{"name":"Schema Registry","displayName":"Schema Registry","identifiesControllerService":true,"sensitive":false,"dynamic":false},"Schema Access Strategy":{"name":"Schema Access Strategy","displayName":"Schema Access Strategy","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Schema Name":{"name":"Schema Name","displayName":"Schema Name","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Schema Write Strategy":{"name":"Schema Write Strategy","displayName":"Schema Write Strategy","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Encoder Pool Size":{"name":"Encoder Pool Size","displayName":"Encoder Pool Size","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Compression Format":{"name":"Compression Format","displayName":"Compression Format","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Schema Cache":{"name":"Schema Cache","displayName":"Schema Cache","identifiesControllerService":true,"sensitive":false,"dynamic":false},"Schema Version":{"name":"Schema Version","displayName":"Schema Version","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Cache Size":{"name":"Cache Size","displayName":"Cache Size","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Schema Reference Writer":{"name":"Schema Reference Writer","displayName":"Schema Reference Writer","identifiesControllerService":true,"sensitive":false,"dynamic":false}},"controllerServiceApis":[{"type":"org.apache.nifi.serialization.RecordSetWriterFactory","bundle":{"group":"org.apache.nifi","artifact":"nifi-standard-services-api-nar","version":"2.10.0"}}],"scheduledState":"DISABLED","bulletinLevel":"WARN","componentType":"CONTROLLER_SERVICE","groupIdentifier":"62dd0a2b-528e-3db2-af01-43a69b2b4110"},{"identifier":"28f235b7-a6af-3e75-b7d5-6fbc32ecf77c","instanceIdentifier":"1d89800d-019f-1000-f19d-d807d7b00440","name":"StandardSSLContextService","comments":"","type":"org.apache.nifi.ssl.StandardSSLContextService","bundle":{"group":"org.apache.nifi","artifact":"nifi-ssl-context-service-nar","version":"2.10.0"},"properties":{"Truststore Type":"PKCS12","Keystore Type":"JKS","Truststore Filename":"/security/certificates/elastic/opensearch/elasticsearch/elasticsearch-1/elasticsearch-1-truststore.key","TLS Protocol":"TLS","Keystore Filename":"/security/certificates/elastic/opensearch/elasticsearch/elasticsearch-1/elasticsearch-1-keystore.jks"},"propertyDescriptors":{"Truststore Type":{"name":"Truststore Type","displayName":"Truststore Type","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Key Password":{"name":"Key Password","displayName":"Key Password","identifiesControllerService":false,"sensitive":true,"dynamic":false},"Keystore Type":{"name":"Keystore Type","displayName":"Keystore Type","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Truststore Filename":{"name":"Truststore Filename","displayName":"Truststore Filename","identifiesControllerService":false,"sensitive":false,"dynamic":false,"resourceDefinition":{"cardinality":"SINGLE","resourceTypes":["FILE"]}},"Keystore Password":{"name":"Keystore Password","displayName":"Keystore Password","identifiesControllerService":false,"sensitive":true,"dynamic":false},"Truststore Password":{"name":"Truststore Password","displayName":"Truststore Password","identifiesControllerService":false,"sensitive":true,"dynamic":false},"TLS Protocol":{"name":"TLS Protocol","displayName":"TLS Protocol","identifiesControllerService":false,"sensitive":false,"dynamic":false},"Keystore Filename":{"name":"Keystore Filename","displayName":"Keystore Filename","identifiesControllerService":false,"sensitive":false,"dynamic":false,"resourceDefinition":{"cardinality":"SINGLE","resourceTypes":["FILE"]}}},"controllerServiceApis":[{"type":"org.apache.nifi.ssl.SSLContextService","bundle":{"group":"org.apache.nifi","artifact":"nifi-standard-services-api-nar","version":"2.10.0"}},{"type":"org.apache.nifi.ssl.SSLContextProvider","bundle":{"group":"org.apache.nifi","artifact":"nifi-standard-services-api-nar","version":"2.10.0"}}],"scheduledState":"DISABLED","bulletinLevel":"WARN","componentType":"CONTROLLER_SERVICE","groupIdentifier":"62dd0a2b-528e-3db2-af01-43a69b2b4110"}],"defaultFlowFileExpiration":"0 sec","defaultBackPressureObjectThreshold":10000,"defaultBackPressureDataSizeThreshold":"1 GB","scheduledState":"ENABLED","executionEngine":"INHERITED","maxConcurrentTasks":1,"statelessFlowTimeout":"1 min","flowFileConcurrency":"UNBOUNDED","flowFileOutboundPolicy":"STREAM_WHEN_AVAILABLE","componentType":"PROCESS_GROUP"},"externalControllerServices":{},"parameterContexts":{},"flowEncodingVersion":"1.0","parameterProviders":{},"latest":false} \ No newline at end of file From 9d86c31abce1a0ad9f71033545b1252c9eaf6bbe Mon Sep 17 00:00:00 2001 From: vladd-bit Date: Wed, 11 Nov 2026 15:28:07 +0000 Subject: [PATCH 26/28] Updated script perms. --- nifi/docker/healthcheck.sh | 0 scripts/tests/test_nifi_runtime.py | 0 scripts/tests/test_postgres_certificates.py | 0 services/nginx/docker-entrypoint.sh | 0 services/nginx/healthcheck.sh | 0 5 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 nifi/docker/healthcheck.sh mode change 100644 => 100755 scripts/tests/test_nifi_runtime.py mode change 100644 => 100755 scripts/tests/test_postgres_certificates.py mode change 100644 => 100755 services/nginx/docker-entrypoint.sh mode change 100644 => 100755 services/nginx/healthcheck.sh diff --git a/nifi/docker/healthcheck.sh b/nifi/docker/healthcheck.sh old mode 100644 new mode 100755 diff --git a/scripts/tests/test_nifi_runtime.py b/scripts/tests/test_nifi_runtime.py old mode 100644 new mode 100755 diff --git a/scripts/tests/test_postgres_certificates.py b/scripts/tests/test_postgres_certificates.py old mode 100644 new mode 100755 diff --git a/services/nginx/docker-entrypoint.sh b/services/nginx/docker-entrypoint.sh old mode 100644 new mode 100755 diff --git a/services/nginx/healthcheck.sh b/services/nginx/healthcheck.sh old mode 100644 new mode 100755 From 835985a2a2cd7c9dc6d1bff46c1d47ded20aa3e1 Mon Sep 17 00:00:00 2001 From: vladd-bit Date: Wed, 11 Nov 2026 15:33:02 +0000 Subject: [PATCH 27/28] Updated script syntax. --- nifi/docker/healthcheck.sh | 0 scripts/tests/test_nifi_runtime.py | 0 scripts/tests/test_postgres_certificates.py | 0 security/scripts/create_postgres_certs.sh | 2 ++ security/scripts/nifi_init_create_user_auth.sh | 2 ++ services/nginx/docker-entrypoint.sh | 0 services/nginx/healthcheck.sh | 0 7 files changed, 4 insertions(+) mode change 100755 => 100644 nifi/docker/healthcheck.sh mode change 100755 => 100644 scripts/tests/test_nifi_runtime.py mode change 100755 => 100644 scripts/tests/test_postgres_certificates.py mode change 100755 => 100644 services/nginx/docker-entrypoint.sh mode change 100755 => 100644 services/nginx/healthcheck.sh diff --git a/nifi/docker/healthcheck.sh b/nifi/docker/healthcheck.sh old mode 100755 new mode 100644 diff --git a/scripts/tests/test_nifi_runtime.py b/scripts/tests/test_nifi_runtime.py old mode 100755 new mode 100644 diff --git a/scripts/tests/test_postgres_certificates.py b/scripts/tests/test_postgres_certificates.py old mode 100755 new mode 100644 diff --git a/security/scripts/create_postgres_certs.sh b/security/scripts/create_postgres_certs.sh index d89064c21..03968c7e7 100755 --- a/security/scripts/create_postgres_certs.sh +++ b/security/scripts/create_postgres_certs.sh @@ -5,7 +5,9 @@ umask 077 # shellcheck disable=SC1090,SC1091 source "$(dirname "${BASH_SOURCE[0]}")/load_config.sh" +# shellcheck disable=SC1090 source "$(cogstack_env_file deploy/database.env)" +# shellcheck disable=SC1090 source "$(cogstack_env_file security/env/certificates_general.env)" resolve_directory() { diff --git a/security/scripts/nifi_init_create_user_auth.sh b/security/scripts/nifi_init_create_user_auth.sh index f30fbce1e..808d933f7 100755 --- a/security/scripts/nifi_init_create_user_auth.sh +++ b/security/scripts/nifi_init_create_user_auth.sh @@ -28,6 +28,8 @@ DEPLOY_DIR="${REPO_ROOT}/deploy" source "${DEPLOY_DIR}/export_env_vars.sh" compose=(docker compose --project-directory "$DEPLOY_DIR" -f "$DEPLOY_DIR/services.yml") +# Expand these variables inside the container, not in the host shell. +# shellcheck disable=SC2016 "${compose[@]}" exec -T nifi /bin/bash -ec \ '"${NIFI_HOME}/bin/nifi.sh" set-single-user-credentials "${NIFI_SINGLE_USER_CREDENTIALS_USERNAME:?}" "${NIFI_SINGLE_USER_CREDENTIALS_PASSWORD:?}"' "${compose[@]}" restart nifi diff --git a/services/nginx/docker-entrypoint.sh b/services/nginx/docker-entrypoint.sh old mode 100755 new mode 100644 diff --git a/services/nginx/healthcheck.sh b/services/nginx/healthcheck.sh old mode 100755 new mode 100644 From c274702e5fa46620a445ee952d292ae04b428068 Mon Sep 17 00:00:00 2001 From: vladd-bit Date: Wed, 11 Nov 2026 15:36:17 +0000 Subject: [PATCH 28/28] fix: mark scripts as executable --- nifi/docker/healthcheck.sh | 0 scripts/tests/test_nifi_runtime.py | 0 scripts/tests/test_postgres_certificates.py | 0 services/nginx/docker-entrypoint.sh | 0 services/nginx/healthcheck.sh | 0 5 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 nifi/docker/healthcheck.sh mode change 100644 => 100755 scripts/tests/test_nifi_runtime.py mode change 100644 => 100755 scripts/tests/test_postgres_certificates.py mode change 100644 => 100755 services/nginx/docker-entrypoint.sh mode change 100644 => 100755 services/nginx/healthcheck.sh diff --git a/nifi/docker/healthcheck.sh b/nifi/docker/healthcheck.sh old mode 100644 new mode 100755 diff --git a/scripts/tests/test_nifi_runtime.py b/scripts/tests/test_nifi_runtime.py old mode 100644 new mode 100755 diff --git a/scripts/tests/test_postgres_certificates.py b/scripts/tests/test_postgres_certificates.py old mode 100644 new mode 100755 diff --git a/services/nginx/docker-entrypoint.sh b/services/nginx/docker-entrypoint.sh old mode 100644 new mode 100755 diff --git a/services/nginx/healthcheck.sh b/services/nginx/healthcheck.sh old mode 100644 new mode 100755