From 28f37ad6053086daf56ec7e89e22cc64bce251c7 Mon Sep 17 00:00:00 2001 From: DefectDojo release bot Date: Thu, 12 Nov 2026 05:01:51 +0000 Subject: [PATCH 01/61] Update versions in application files --- components/package.json | 2 +- dojo/__init__.py | 2 +- helm/defectdojo/Chart.yaml | 8 ++++---- helm/defectdojo/README.md | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/components/package.json b/components/package.json index a4228b15cb4..e1213fd09ea 100644 --- a/components/package.json +++ b/components/package.json @@ -1,6 +1,6 @@ { "name": "defectdojo", - "version": "3.5.102", + "version": "3.6.0-dev", "license": "BSD-3-Clause", "private": true, "dependencies": { diff --git a/dojo/__init__.py b/dojo/__init__.py index f48ba085353..806be4028c3 100644 --- a/dojo/__init__.py +++ b/dojo/__init__.py @@ -4,6 +4,6 @@ # Django starts so that shared_task will use this app. from .celery import app as celery_app # noqa: F401 -__version__ = "3.5.102" +__version__ = "3.6.0-dev" __url__ = "https://github.com/DefectDojo/django-DefectDojo" __docs__ = "https://documentation.defectdojo.com" diff --git a/helm/defectdojo/Chart.yaml b/helm/defectdojo/Chart.yaml index 8375bd3af5b..8fab64082fe 100644 --- a/helm/defectdojo/Chart.yaml +++ b/helm/defectdojo/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v2 -appVersion: "3.5.102" +appVersion: "3.6.0-dev" description: A Helm chart for Kubernetes to install DefectDojo name: defectdojo -version: 1.9.68 +version: 1.9.69-dev icon: https://defectdojo.com/hubfs/DefectDojo_favicon.png maintainers: - name: madchap @@ -33,5 +33,5 @@ dependencies: # - kind: security # description: Critical bug annotations: - artifacthub.io/prerelease: "false" - artifacthub.io/changes: "- kind: changed\n description: Bump DefectDojo to 3.5.102\n" + artifacthub.io/prerelease: "true" + artifacthub.io/changes: "" diff --git a/helm/defectdojo/README.md b/helm/defectdojo/README.md index b8397ba770a..ae5b9b02eb8 100644 --- a/helm/defectdojo/README.md +++ b/helm/defectdojo/README.md @@ -521,7 +521,7 @@ The HELM schema will be generated for you. # General information about chart values -![Version: 1.9.68](https://img.shields.io/badge/Version-1.9.68-informational?style=flat-square) ![AppVersion: 3.5.102](https://img.shields.io/badge/AppVersion-3.5.102-informational?style=flat-square) +![Version: 1.9.69-dev](https://img.shields.io/badge/Version-1.9.69--dev-informational?style=flat-square) ![AppVersion: 3.6.0-dev](https://img.shields.io/badge/AppVersion-3.6.0--dev-informational?style=flat-square) A Helm chart for Kubernetes to install DefectDojo From 39bf6da85f5b98485fb633598323567bd2bcf542 Mon Sep 17 00:00:00 2001 From: Blake Owens <76979297+blakeaowens@users.noreply.github.com> Date: Thu, 12 Nov 2026 05:48:55 +0000 Subject: [PATCH 02/61] docs(rules-engine): say a rule close mitigates the finding's endpoints (#16802) * docs(rules-engine): say a rule close mitigates the finding's endpoints Set Status (false_positive, mitigated), Copy Status from Matched Finding and the classic Rules Engine's False Positive / Mitigated fields mitigate the closed Finding's open Endpoints (or Locations), as closing it by hand does. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HJqD5GHgqGFiHWpR4JYfWm * docs(rules-engine): say a rule reopen opens the endpoints again A rule reopening a Finding sets its Endpoints (or Locations) open again; the close names the rule owner and the run's time. Folds the classic Rules Engine bullet into one sentence and adds the sentence to the ticket sync node. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HJqD5GHgqGFiHWpR4JYfWm --------- Co-authored-by: Claude Opus 5.5 --- docs/content/automation/rules_engine/about.md | 2 +- docs/content/automation/triage_engine/node_reference.md | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/content/automation/rules_engine/about.md b/docs/content/automation/rules_engine/about.md index 614cb9b184e..5e1f129e546 100644 --- a/docs/content/automation/rules_engine/about.md +++ b/docs/content/automation/rules_engine/about.md @@ -26,7 +26,7 @@ Rules can be triggered manually from the **Classic Rules** page, or scheduled to Each Rule can apply one or more of these changes to a Finding when it is triggered successfully (i.e. matches the set Filter conditions). ### Field Modifications -* **Set a field** on a Finding, including Title, Description, Severity, CVSSv3 Vector, Active, Verified, Risk Accepted, False Positive, Mitigated. Setting False Positive or Mitigated on a risk-accepted Finding takes it out of its risk acceptance first, as closing it by hand does. +* **Set a field** on a Finding, including Title, Description, Severity, CVSSv3 Vector, Active, Verified, Risk Accepted, False Positive, Mitigated. Setting False Positive or Mitigated works as closing the Finding by hand does: a risk-accepted Finding leaves its risk acceptance first, and the Finding's open Endpoints (or Locations) are mitigated. Setting Active on a closed Finding, or clearing Mitigated, sets its Endpoints (or Locations) open again, as reopening it by hand does. * **Append or Prepend text** to a Finding's Title or Description * **Set Priority** — override the calculated Priority value on a Finding (overrides automatic priority calculation) * **Set Risk** — override the calculated Risk level on a Finding (overrides automatic risk calculation) diff --git a/docs/content/automation/triage_engine/node_reference.md b/docs/content/automation/triage_engine/node_reference.md index fe4549bf6f2..30cf76b6f74 100644 --- a/docs/content/automation/triage_engine/node_reference.md +++ b/docs/content/automation/triage_engine/node_reference.md @@ -388,6 +388,8 @@ Moves the Finding to a status. `false_positive` and `mitigated` take a risk-accepted Finding out of its risk acceptance first, as closing it by hand does: it leaves the acceptance, and a note on the Finding says so. +As closing it by hand does, `false_positive` and `mitigated` also mitigate the Finding's open Endpoints (or, with Locations on, set its Locations to Mitigated), naming the rule owner and the time of the run. An Endpoint already mitigated keeps its own date. `reopen`, and `active` on a closed Finding, set its Endpoints (or Locations) open again, as reopening it by hand does. A preview changes nothing. + ### Copy Status from Matched Finding `finding.copy_matched_status` @@ -402,6 +404,8 @@ Gives the Finding the status of the Finding deduplication matched it to in anoth A status is read in this order: false positive, out of scope, risk accepted, mitigated, and otherwise open. A risk acceptance is carried as a new acceptance on the Finding's own engagement, with the same expiration date, so either one can later be extended or revoked on its own. If the Asset only allows simple risk acceptance, the Finding is accepted the simple way, which has no expiration. The rule owner must be able to see the Finding the status comes from (the note names it) and edit the Finding it goes to, and to add a risk acceptance for that status; anything else is left alone, and so are duplicates. It does nothing after **On Dedupe Match** in **Mark as Duplicate** mode, because the new Finding is then a duplicate. A Finding already closed for a different reason takes the new status in place of the old one, so the two Findings end up with the same status. +Carrying `False Positive`, `Out of Scope` or `Mitigated` also mitigates the Finding's open Endpoints (or, with Locations on, sets its Locations to Mitigated), as closing it by hand does, and carrying `Reopened` sets them open again. + ### Apply Status to Linked Findings `finding.apply_status_to_linked` @@ -458,7 +462,7 @@ Closes, reopens, false-positives or risk accepts each Finding to match its linke A close reason only ever classifies a closed state: false positive first, then accepted risk, then plain mitigation. A reopened ticket that still carries its old resolution reopens. Findings already in the target state are left alone. -An accepted risk works the way the classic Jira webhook does: it creates a full risk acceptance where the Finding's product allows one, a simple risk acceptance where only that is allowed, and otherwise closes the Finding as mitigated. Reopening or false-positiving a risk-accepted Finding removes its risk acceptance first. A status change is only made on a Finding the rule owner may edit (and, for a risk acceptance, may accept risk on). +An accepted risk works the way the classic Jira webhook does: it creates a full risk acceptance where the Finding's product allows one, a simple risk acceptance where only that is allowed, and otherwise closes the Finding as mitigated. Reopening or false-positiving a risk-accepted Finding removes its risk acceptance first. Closing a Finding mitigates its open Endpoints (or Locations), and reopening it sets them open again, as closing and reopening it by hand do. A status change is only made on a Finding the rule owner may edit (and, for a risk acceptance, may accept risk on). ### Add a Ticket Comment as a Note From 5506223bae6b3925adcd54da85162fd357bcdc91 Mon Sep 17 00:00:00 2001 From: Blake Owens <76979297+blakeaowens@users.noreply.github.com> Date: Thu, 12 Nov 2026 08:46:37 +0000 Subject: [PATCH 03/61] docs(risk-acceptance): say acceptances from before 3.3.0 get their finding records on upgrade (#16808) * docs(risk-acceptance): say acceptances from before 3.3.0 get their finding records on upgrade Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GLL1WfyvvMFAmudbcCyaZK * docs(risk-acceptance): an expiry does not sever an acceptance's membership Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- .../findings_workflows/PRO__risk_acceptance.md | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/docs/content/triage_findings/findings_workflows/PRO__risk_acceptance.md b/docs/content/triage_findings/findings_workflows/PRO__risk_acceptance.md index e615b12139a..4a5fcc39896 100644 --- a/docs/content/triage_findings/findings_workflows/PRO__risk_acceptance.md +++ b/docs/content/triage_findings/findings_workflows/PRO__risk_acceptance.md @@ -465,9 +465,15 @@ records: - a per-Finding justification, separate from the acceptance's overall recommendation — auditors ask about individual Findings, and the answer is rarely the same for every Finding in a batch. -Records are not backfilled. A Risk Acceptance created before you enabled the feature can be seeded -from the Findings it still covers, but Findings whose membership was already severed cannot be -recovered — that history was not kept. +Risk Acceptances that existed before DefectDojo Pro 3.3.0, when records began, get them on +upgrade: every Finding a Risk Acceptance still covers is recorded with the source `backfill`, with +the status the acceptance has now (an expired acceptance's Findings read as expired), and with the +date the acceptance was created as the date it took effect. Nobody is recorded as having added +those Findings, because nobody was on record. Findings that already have a record keep it as it is. + +Findings whose membership was severed before 3.3.0 (by a re-import, or a person removing them) +cannot be recovered: that history was never kept, and the backfill can only see the Findings that +are still members. ### Standing acceptances From 3da50f8329ca866e757c591a83336ba0710ed1d0 Mon Sep 17 00:00:00 2001 From: Blake Owens <76979297+blakeaowens@users.noreply.github.com> Date: Thu, 12 Nov 2026 09:15:52 +0000 Subject: [PATCH 04/61] docs(rules-engine): say the Set a Field pickers list the instance's own options (#16806) Claude-Session: https://claude.ai/code/session_01HJqD5HCezZb93pC6XgZHcT Co-authored-by: Claude Opus 5.5 --- docs/content/automation/triage_engine/node_reference.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/content/automation/triage_engine/node_reference.md b/docs/content/automation/triage_engine/node_reference.md index 30cf76b6f74..658aa5b4a73 100644 --- a/docs/content/automation/triage_engine/node_reference.md +++ b/docs/content/automation/triage_engine/node_reference.md @@ -631,9 +631,9 @@ Sets a field on the Asset. Assets already holding the value are left alone, and | Setting | Default | Notes | |---------|---------|-------| | **Field** | none | One of: `description`, `business_criticality`, `platform`, `lifecycle`, `origin`, `user_records`, `revenue`, `external_audience`, `internet_accessible`, `asset_type`. | -| **Value** | none | The control follows the field: a template for description (with `{{product.name}}` style placeholders), a picker for the choice fields, a number or a toggle for the rest. | +| **Value** | none | The control follows the field: a template for description (with `{{product.name}}` style placeholders), a picker for the choice fields, a number or a toggle for the rest. The `platform`, `lifecycle`, `origin` and `asset_type` pickers list the options configured on the instance, with their current names. | -For `asset_type`, the value is a type's code (`host`, `repository`, `service`, ...). The picker lists the types DefectDojo ships; a saved rule may name a type you added, and a code that names no type fails the node. A type a rule sets counts as chosen by a person, so connectors do not overwrite it. In conditions, the Asset's type is `product.kind`. +For `asset_type`, the value is a type's code (`host`, `repository`, `service`, ...). The picker lists every type on the instance, the ones you added included, under their current labels and in their display order. A code that names no type fails the node. A type a rule sets counts as chosen by a person, so connectors do not overwrite it. In conditions, the Asset's type is `product.kind`. Two fields are deliberately not offered: `name` (it is unique, so one rendered value written across a sweep cannot work) and the SLA configuration (it drives an asynchronous recalculation with its own write lock, and belongs to a dedicated flow). From fe0bb37b224aa555cbd74661cdd3b9b108993c24 Mon Sep 17 00:00:00 2001 From: Blake Owens <76979297+blakeaowens@users.noreply.github.com> Date: Thu, 12 Nov 2026 09:21:48 +0000 Subject: [PATCH 05/61] fix(findings): match a lifecycle value that contains a comma, in quotes (#16790) * fix(product-attributes): refuse a comma in a new attribute option value The findings filters over Asset attribute values (product_lifecycle on /api/v2/findings/) take a comma-separated list, so an option whose value contains a comma could never be matched. Creating a Platform, Lifecycle or Origin option through /api/v2/ or the classic UI form now refuses a value with a comma. Only a create is checked: an update ignores the value, and an existing row keeps its own, so a legacy row stays editable. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HJqD5F8fqQRckGxJ6sobkn * fix(product-attributes): translate the comma refusal and check it on the model - The message is gettext_lazy, so /api/v2/ and the classic form show it in the user's language. - The create-only check moves from the classic form to the model's clean(), so every ModelForm, Django admin included, refuses a comma in a new value; an existing row keeps its own. The API serializers keep their own check. - The docs say a derived value also collapses repeated spaces. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HJqD5HBSaAhHF6FQrzAbmo * fix(findings): match a lifecycle value that contains a comma, in quotes Replaces the comma refusal on Asset attribute option values with a filter that can match them. An option's value may contain a comma, but product_lifecycle on /api/v2/findings/ split its parameter on every comma, so such an option could never be matched. product_lifecycle now reads a value in double quotes as one value, CSV-style ("beta, ga",production); a list without quotes splits exactly as before. The create-time refusal, its model check and their test are reverted, and the attribute options docs say how to quote such a value. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HJqD5F8fqQRckGxJ6sobkn --------- Co-authored-by: Claude Opus 5.5 --- .../PRO__asset_attribute_options.md | 4 +- dojo/filters.py | 9 ++++ dojo/finding/api/filters.py | 3 +- .../test_findings_product_lifecycle_filter.py | 50 +++++++++++++++++++ 4 files changed, 64 insertions(+), 2 deletions(-) create mode 100644 unittests/test_findings_product_lifecycle_filter.py diff --git a/docs/content/asset_modelling/PRO__asset_attribute_options.md b/docs/content/asset_modelling/PRO__asset_attribute_options.md index ccaada8a098..e8b121d99f1 100644 --- a/docs/content/asset_modelling/PRO__asset_attribute_options.md +++ b/docs/content/asset_modelling/PRO__asset_attribute_options.md @@ -19,7 +19,9 @@ lists the current options with: is the part you edit. - **Value** — a fixed machine key stored on the asset and used by the API, imports and automation rules. It is set when an option is created and never changes afterward, so relabeling an option - never breaks an integration. + never breaks an integration. A value may contain a comma: the findings filters that take a + comma-separated list of values (`product_lifecycle` on `/api/v2/findings/`) read a value in + double quotes as one value, for example `"beta, ga",production`. - **Icon** and **Display order** — optional. The icon is a Font Awesome name; display order controls where the option appears in the dropdown. - **Assets Using** — how many assets currently reference the option. diff --git a/dojo/filters.py b/dojo/filters.py index 79e00084507..9a06431fcc7 100644 --- a/dojo/filters.py +++ b/dojo/filters.py @@ -1,4 +1,5 @@ import collections +import csv import decimal import logging from datetime import datetime, timedelta @@ -73,6 +74,14 @@ def custom_filter(queryset, name, value): return queryset.filter(Q(**{cust_filter: values})) +def quoted_list_filter(queryset, name, value): + # custom_filter for values that may hold a comma (an Asset attribute option's value): such a + # value travels in double quotes, CSV-style (``"beta, ga",production``). A list without quotes + # splits on every comma exactly as custom_filter does. + values = next(csv.reader([value]), []) + return queryset.filter(Q(**{f"{name}__in": values})) + + def custom_vulnerability_id_filter(queryset, name, value): values = value.split(",") return queryset.filter(id__in=finding_ids_with_vulnerability_ids(values, lookup="in")) diff --git a/dojo/finding/api/filters.py b/dojo/finding/api/filters.py index 446d51fe10f..bbc6188d808 100644 --- a/dojo/finding/api/filters.py +++ b/dojo/finding/api/filters.py @@ -26,6 +26,7 @@ custom_filter, custom_vulnerability_id_filter, labels, + quoted_list_filter, ) from dojo.models import Finding, Finding_Template @@ -66,7 +67,7 @@ class ApiFindingFilter(DojoFilter): exact_title = CharFilter(field_name="title", lookup_expr="iexact", help_text="Finding title exact match (case-insensitive)") product_name = CharFilter(lookup_expr="engagement__product__name__iexact", field_name="test", label=labels.ASSET_FILTERS_NAME_EXACT_LABEL) product_name_contains = CharFilter(lookup_expr="engagement__product__name__icontains", field_name="test", label=labels.ASSET_FILTERS_NAME_CONTAINS_LABEL) - product_lifecycle = CharFilter(method=custom_filter, lookup_expr="engagement__product__lifecycle__value", + product_lifecycle = CharFilter(method=quoted_list_filter, lookup_expr="engagement__product__lifecycle__value", field_name="test__engagement__product__lifecycle__value", label=labels.ASSET_FILTERS_CSV_LIFECYCLES_LABEL) # DateRangeFilter created = DateRangeFilter() diff --git a/unittests/test_findings_product_lifecycle_filter.py b/unittests/test_findings_product_lifecycle_filter.py new file mode 100644 index 00000000000..5fe6fe1e703 --- /dev/null +++ b/unittests/test_findings_product_lifecycle_filter.py @@ -0,0 +1,50 @@ +from django.test import override_settings +from django.utils.timezone import now +from rest_framework.test import APITestCase + +from dojo.models import Dojo_User, Engagement, Finding, Product, Product_Lifecycle, Product_Type, Test, Test_Type + + +@override_settings(SECURE_SSL_REDIRECT=False) +class TestFindingsProductLifecycleFilter(APITestCase): + + """ + Regression: a Lifecycle option's value may contain a comma (``beta, ga``), but ``product_lifecycle`` + split its parameter on every comma, so such an option could never be matched. A value holding a + comma now travels in double quotes, CSV-style (``"beta, ga",production``); a list without quotes + splits exactly as before. + """ + + @classmethod + def setUpTestData(cls): + cls.user = Dojo_User.objects.create(username="lifecycle_filter_admin", is_staff=True, is_superuser=True) + prod_type = Product_Type.objects.create(name="Lifecycle Filter PT") + test_type = Test_Type.objects.create(name="Lifecycle Filter Scan") + cls.findings = {} + for value in ("beta, ga", "production", "beta"): + lifecycle, _ = Product_Lifecycle.objects.get_or_create(value=value, defaults={"name": value}) + product = Product.objects.create(name=f"Lifecycle Filter {value}", description="x", prod_type=prod_type, lifecycle=lifecycle) + engagement = Engagement.objects.create(name="e", product=product, target_start=now(), target_end=now()) + test = Test.objects.create(engagement=engagement, test_type=test_type, target_start=now(), target_end=now()) + cls.findings[value] = Finding.objects.create( + title=f"Lifecycle Filter {value}", test=test, severity="High", reporter=cls.user, date=now().date(), + ).id + + def setUp(self): + self.client.force_authenticate(user=self.user) + + def _values(self, param): + response = self.client.get("/api/v2/findings/", {"product_lifecycle": param, "limit": 100}) + self.assertEqual(response.status_code, 200, response.content[:500]) + by_id = {finding_id: value for value, finding_id in self.findings.items()} + return {by_id[row["id"]] for row in response.json()["results"] if row["id"] in by_id} + + def test_a_quoted_value_with_a_comma_is_one_value(self): + self.assertEqual(self._values('"beta, ga"'), {"beta, ga"}) + + def test_a_quoted_value_sits_in_a_list_with_plain_ones(self): + self.assertEqual(self._values('"beta, ga",production'), {"beta, ga", "production"}) + + def test_a_list_without_quotes_splits_as_before(self): + self.assertEqual(self._values("production,beta"), {"production", "beta"}) + self.assertEqual(self._values("production"), {"production"}) From 1273c3d217d3fa786420ddd5182a49796e1af170 Mon Sep 17 00:00:00 2001 From: Greg Anderson Date: Thu, 12 Nov 2026 11:46:12 +0000 Subject: [PATCH 06/61] Add Early Light parser (#16674) * feat(parsers): add Early Light JSON parser Adds the "Early Light Scan" scan type for `earlylight scan --format json` reports from DefectDojo's emerging-vulnerability scanner. - vulnerable findings are imported tagged "earlylight"; unknown findings are imported tagged "earlylight" and "needs-review" with the reason in the description; fixed, inactive and managed findings are skipped - vuln_id_from_tool is "/"; vulnerability IDs are the finding's own CVEs only - severity is the finding's own when set, else the kind's severity from the detector catalog, else High (vulnerable) or Medium (unknown) - unique_id_from_tool is a stable hash of detector, kind, target, path and the certificate or image facts; deduplication uses unique_id_from_tool - sample reports are real Early Light output, trimmed Co-Authored-By: Claude Opus 5.5 * feat(parsers): prefer the kind severities newer Early Light reports carry Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- .../parsers/file/earlylight.md | 63 ++ dojo/settings/settings.dist.py | 3 + dojo/tools/earlylight/__init__.py | 0 dojo/tools/earlylight/parser.py | 285 +++++++++ unittests/scans/earlylight/cleared_only.json | 252 ++++++++ unittests/scans/earlylight/many_targets.json | 546 ++++++++++++++++++ unittests/scans/earlylight/many_vulns.json | 264 +++++++++ unittests/scans/earlylight/needs_review.json | 165 ++++++ unittests/scans/earlylight/no_findings.json | 55 ++ unittests/scans/earlylight/one_vuln.json | 129 +++++ unittests/scans/earlylight/repo.json | 112 ++++ unittests/tools/test_earlylight_parser.py | 257 +++++++++ 12 files changed, 2131 insertions(+) create mode 100644 docs/content/supported_tools/parsers/file/earlylight.md create mode 100644 dojo/tools/earlylight/__init__.py create mode 100644 dojo/tools/earlylight/parser.py create mode 100644 unittests/scans/earlylight/cleared_only.json create mode 100644 unittests/scans/earlylight/many_targets.json create mode 100644 unittests/scans/earlylight/many_vulns.json create mode 100644 unittests/scans/earlylight/needs_review.json create mode 100644 unittests/scans/earlylight/no_findings.json create mode 100644 unittests/scans/earlylight/one_vuln.json create mode 100644 unittests/scans/earlylight/repo.json create mode 100644 unittests/tools/test_earlylight_parser.py diff --git a/docs/content/supported_tools/parsers/file/earlylight.md b/docs/content/supported_tools/parsers/file/earlylight.md new file mode 100644 index 00000000000..de25c0cd77f --- /dev/null +++ b/docs/content/supported_tools/parsers/file/earlylight.md @@ -0,0 +1,63 @@ +--- +title: "Early Light" +toc_hide: true +--- +### File Types +[Early Light](https://github.com/DefectDojo/earlylight) is DefectDojo's open source emerging-vulnerability scanner. Each detector targets one newly disclosed vulnerability and checks filesystems, container images, source trees or live hosts for it. + +This parser imports the JSON report. Generate it with: + +``` +earlylight scan fs --format json --output earlylight.json /path/to/dir +earlylight scan image --format json --output earlylight.json php:8.3.0-cli +earlylight scan repo --format json --output earlylight.json /path/to/source +earlylight scan host --format json --output earlylight.json example.com:443 +``` + +Flags go before the paths, image references or hosts. One report can hold several targets (for example several images scanned in one run), and the parser imports the findings of all of them. + +### Which findings are imported +Early Light gives every finding a verdict: + +- **vulnerable**: imported, tagged `earlylight`. +- **unknown**: imported, tagged `earlylight` and `needs-review`. The description starts with the reason Early Light could not settle it. +- **fixed**, **inactive** and **managed**: Early Light cleared these, so they are not imported. + +A report with no vulnerable or unknown findings imports no findings. + +### Field Mapping Details +- **title**: `/: `, for example `php2shell/php-runtime-web: PHP 8.3.0 (cgi-fcgi)`. +- **vuln_id_from_tool**: `/`, for example `brotli/bundled-decoder`. +- **vulnerability_ids**: the CVE IDs listed on the finding itself (`cves[].id`). The detector's own CVE list is not copied, so a finding with no CVEs of its own, such as an eMudhra certificate trust finding, has none. +- **description**: the verdict or the needs-review reason, the message, the target (image reference, host or path, with its type, digest and platform), the path and line, the detection method, the detector, every fact, and the verdict of each CVE with its reason. +- **mitigation**: the finding's `fix`. `fix_available` is set when there is one. +- **references**: the detector's advisory links. +- **file_path** and **line**: the finding's `path` and `line` (the line is set only for findings in source files). +- **component_name** and **component_version**: from the facts when present: `product` and `version` (a program or library carrying the Brotli decoder), `package` and `current` (a dependency that bundles it), the installing OS package (`package`), or `php` and `php_version`. +- **cwe**: the first CWE of the kind, from the detector catalog. +- **static_finding** / **dynamic_finding**: host scans are dynamic, all other scans are static. + +### Severity +1. If the finding states its own severity (`critical`, `high`, `medium` or `low`), that is used. php2shell findings do this. The values `review` and `none` that php2shell also writes are not severities and fall through to the next rule. +2. Otherwise the severity of the kind is used. The JSON report does not carry it, so the parser keeps a table built from the detector catalog (`earlylight detectors --format json`): a CVSS-style score of 9 or more is Critical, 7 or more High, 4 or more Medium, anything lower Low. A kind without a score uses its level: error is High, warning is Medium, note is Info. + +| Kind | Severity | +|------|----------| +| `brotli/decoder-library`, `brotli/bundled-decoder`, `brotli/node-pin`, `brotli/dotnet-pin`, `brotli/bun-pin`, `brotli/dependency` | Critical (9.2) | +| `brotli/go-port` | Medium (5.3) | +| `brotli/not-checked` | Info (note) | +| `emudhra/compromised-key`, `emudhra/path-root`, `emudhra/weak-key`, `emudhra/issued-by-compromised-ca` | High (error) | +| `emudhra/unpatched-go` | Medium (warning) | +| `php2shell/php-runtime-web`, `php2shell/php-image` | Critical (9.2) | +| `php2shell/php-runtime-cli` | High (7.5) | + +3. A kind the parser does not know yet (from a detector released after it) is High when vulnerable and Medium when unknown. + +### Deduplication +The "Early Light Scan" scan type uses the `unique_id_from_tool` [deduplication algorithm](https://docs.defectdojo.com/en/working_with_findings/finding_deduplication/about_deduplication/). The parser sets `unique_id_from_tool` to a SHA-256 of the detector, the kind, the target type, the target, the path, and the certificate (`spki_sha256`, `cert_sha256`) or image reference (`image`) facts when present. It does not include line numbers or wording, so the same finding keeps its ID across scans. Findings that would still be identical in one report are told apart by their order. + +### Sample Scan Data +Sample scan data for testing purposes can be found [here](https://github.com/DefectDojo/django-DefectDojo/tree/master/unittests/scans/earlylight). + +### Link To Tool +- [Early Light](https://github.com/DefectDojo/earlylight) diff --git a/dojo/settings/settings.dist.py b/dojo/settings/settings.dist.py index be0e1bbd413..996af938022 100644 --- a/dojo/settings/settings.dist.py +++ b/dojo/settings/settings.dist.py @@ -2008,6 +2008,9 @@ def generate_url(scheme, double_slashes, user, password, host, port, path, param # that key rewrites itself as time passes even though the report never changed. "Xeol Parser": DEDUPE_ALGO_HASH_CODE, "OPF Scan": DEDUPE_ALGO_HASH_CODE, + # The parser builds a stable unique_id_from_tool for every finding (detector, kind, target, path and the + # certificate or image that tells findings at one path apart), so no hash_code fallback is needed. + "Early Light Scan": DEDUPE_ALGO_UNIQUE_ID_FROM_TOOL, } # Override the hardcoded settings here via the env var diff --git a/dojo/tools/earlylight/__init__.py b/dojo/tools/earlylight/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dojo/tools/earlylight/parser.py b/dojo/tools/earlylight/parser.py new file mode 100644 index 00000000000..4c749c0b671 --- /dev/null +++ b/dojo/tools/earlylight/parser.py @@ -0,0 +1,285 @@ +""" +Parser for Early Light JSON reports. + +Early Light (https://github.com/DefectDojo/earlylight) is DefectDojo's open source +emerging-vulnerability scanner: one detector per vulnerability, run against +filesystems, container images, source trees and live hosts. This parser reads +the report written by ``earlylight scan --format json``. + +Verdicts: + +- ``vulnerable``: imported, tagged ``earlylight``. +- ``unknown``: imported, tagged ``earlylight`` and ``needs-review``; the reason + Early Light could not settle it is in the description. +- ``fixed``, ``inactive`` and ``managed``: cleared by Early Light, not imported. +""" + +import hashlib +import json + +from dojo.models import Finding + +SCAN_TYPE = "Early Light Scan" + +TAG = "earlylight" +NEEDS_REVIEW_TAG = "needs-review" + +IMPORTED_VERDICTS = {"vulnerable", "unknown"} + +# Severity a finding states for itself (php2shell sets one; "review" and "none" +# are not severities and fall through to the table below). +FINDING_SEVERITIES = { + "critical": "Critical", + "high": "High", + "medium": "Medium", + "low": "Low", + "info": "Info", + "informational": "Info", +} + +# Severity and CWE of each "/", taken from the detector manifests +# (`earlylight detectors --format json`). Reports from Early Light releases after +# v0.3.1 list each detector's kinds with their score and level, and those win; +# this table covers older reports and supplies the CWE. The kind's CVSS-style +# "severity" score maps to >=9 Critical, >=7 High, >=4 Medium, else Low. Kinds +# without a score use their SARIF level: error is High, warning is Medium, note +# is Info. The CWE is the first one the kind lists. +KIND_SEVERITY = { + # brotli: CVE-2026-111497 + "brotli/decoder-library": ("Critical", 252), # 9.2 + "brotli/bundled-decoder": ("Critical", 252), # 9.2 + "brotli/go-port": ("Medium", 252), # 5.3 + "brotli/node-pin": ("Critical", 252), # 9.2 + "brotli/dotnet-pin": ("Critical", 252), # 9.2 + "brotli/bun-pin": ("Critical", 252), # 9.2 + "brotli/dependency": ("Critical", 252), # 9.2 + "brotli/not-checked": ("Info", None), # level note + # emudhra: CVE-2026-111943 (trust kinds carry no CVE of their own) + "emudhra/compromised-key": ("High", None), # level error + "emudhra/path-root": ("High", None), # level error + "emudhra/weak-key": ("High", None), # level error + "emudhra/unpatched-go": ("Medium", None), # level warning + "emudhra/issued-by-compromised-ca": ("High", None), # level error + # php2shell: CVE-2026-109618, CVE-2026-109619 + "php2shell/php-runtime-web": ("Critical", 416), # 9.2 + "php2shell/php-runtime-cli": ("High", 416), # 7.5 + "php2shell/php-image": ("Critical", 416), # 9.2 +} + +LEVEL_SEVERITY = {"error": "High", "warning": "Medium", "note": "Info"} + +# Kinds neither the report nor this table describe. +UNKNOWN_KIND_SEVERITY = {"vulnerable": "High", "unknown": "Medium"} + +# Facts that tell two findings of one kind at one path apart (two certificates +# in one bundle, two images in one file). +IDENTITY_FACTS = ("spki_sha256", "cert_sha256", "image") + +TITLE_MAX = 500 + + +class EarlylightParser: + + """Early Light JSON report (``earlylight scan ... --format json``).""" + + def get_scan_types(self): + return [SCAN_TYPE] + + def get_label_for_scan_types(self, scan_type): + return SCAN_TYPE + + def get_description_for_scan_types(self, scan_type): + return ( + "Early Light emerging-vulnerability scanner JSON report. Generate it with " + "'earlylight scan --format json ...'. Vulnerable findings are imported, " + "unknown ones are imported tagged needs-review, and findings Early Light cleared are skipped." + ) + + def get_findings(self, file, test): + content = file.read() + if isinstance(content, bytes): + content = content.decode("utf-8") + try: + data = json.loads(content) + except ValueError as e: + msg = f"Invalid Early Light JSON report: {e}" + raise ValueError(msg) from e + if not isinstance(data, dict) or not isinstance(data.get("targets"), list): + msg = "Invalid Early Light JSON report: expected an object with a 'targets' list" + # ValueError, not TypeError: the importer turns ValueError into a validation error for the user + raise ValueError(msg) # noqa: TRY004 + + detectors = {d.get("id"): d for d in data.get("detectors") or [] if isinstance(d, dict)} + findings = [] + seen = {} + for target in data["targets"]: + if not isinstance(target, dict): + continue + for item in target.get("findings") or []: + if not isinstance(item, dict) or item.get("verdict") not in IMPORTED_VERDICTS: + continue + finding = self._build_finding(item, target, detectors.get(item.get("detector")) or {}, test) + finding.unique_id_from_tool = self._unique_id(item, target, seen) + findings.append(finding) + return findings + + def _build_finding(self, item, target, detector, test): + detector_id = item.get("detector") or "" + kind = item.get("kind") or "" + rule = f"{detector_id}/{kind}" + verdict = item["verdict"] + facts = item.get("facts") or {} + path = item.get("path") or None + message = item.get("message") or "" + known_severity, cwe = KIND_SEVERITY.get(rule, (None, None)) + known_severity = self._reported_kind_severity(detector, kind) or known_severity + + title = f"{rule}: {message}" if message else rule + if len(title) > TITLE_MAX: + title = title[: TITLE_MAX - 3] + "..." + + is_host = target.get("type") == "host" + finding = Finding( + test=test, + title=title, + severity=self._severity(item, known_severity), + description=self._description(item, target, detector), + mitigation=item.get("fix") or None, + references="\n".join(detector.get("advisories") or []) or None, + file_path=path, + static_finding=not is_host, + dynamic_finding=is_host, + vuln_id_from_tool=rule, + ) + if cwe: + finding.cwe = cwe + line = item.get("line") + if isinstance(line, int) and line > 0: + finding.line = line + if item.get("fix"): + finding.fix_available = True + component_name, component_version = self._component(facts) + if component_name: + finding.component_name = component_name + if component_version: + finding.component_version = component_version + + # Only the finding's own CVEs, never the detector's list: an eMudhra trust + # finding has no CVE even though the detector names one. + cves = [] + for cve in item.get("cves") or []: + cve_id = cve.get("id") if isinstance(cve, dict) else None + if cve_id and cve_id not in cves: + cves.append(cve_id) + if cves: + finding.unsaved_vulnerability_ids = cves + + finding.unsaved_tags = [TAG, NEEDS_REVIEW_TAG] if verdict == "unknown" else [TAG] + return finding + + def _severity(self, item, known_severity): + stated = FINDING_SEVERITIES.get(str(item.get("severity") or "").lower()) + if stated: + return stated + if known_severity: + return known_severity + return UNKNOWN_KIND_SEVERITY[item["verdict"]] + + def _reported_kind_severity(self, detector, kind): + for k in detector.get("kinds") or []: + if not isinstance(k, dict) or k.get("id") != kind: + continue + try: + score = float(k.get("severity") or "") + except ValueError: + return LEVEL_SEVERITY.get(k.get("level")) + if score >= 9: + return "Critical" + if score >= 7: + return "High" + if score >= 4: + return "Medium" + return "Low" + return None + + def _component(self, facts): + if facts.get("product"): + # brotli: the program or library carrying the decoder + return facts["product"], facts.get("version") + if facts.get("ecosystem") and facts.get("package"): + # brotli: a dependency that bundles the decoder + return facts["package"], facts.get("current") + if facts.get("package"): + # php2shell: the OS package that installed PHP, "name version" + name, _, version = facts["package"].partition(" ") + return name, version or None + if facts.get("php_version"): + return "php", facts["php_version"] + return None, None + + def _description(self, item, target, detector): + verdict = item["verdict"] + lines = [] + if verdict == "unknown": + lines.extend([ + "**Needs review:** Early Light could not settle this finding.", + f"**Reason:** {item.get('reason') or 'not given'}", + ]) + else: + lines.append("**Verdict:** vulnerable") + if item.get("reason"): + lines.append(f"**Reason:** {item['reason']}") + if item.get("message"): + lines.append(f"**Message:** {item['message']}") + + target_parts = [target.get("type"), target.get("digest"), target.get("platform")] + target_details = ", ".join(p for p in target_parts if p) + target_line = f"**Target:** {target.get('target') or ''}" + if target_details: + target_line += f" ({target_details})" + lines.append(target_line) + + if item.get("path"): + where = item["path"] + if isinstance(item.get("line"), int) and item["line"] > 0: + where += f":{item['line']}" + lines.append(f"**Path:** {where}") + if item.get("method"): + lines.append(f"**Method:** {item['method']}") + + detector_line = f"**Detector:** {item.get('detector')}/{item.get('kind')}" + if detector.get("title"): + detector_line += f" ({detector['title']})" + lines.append(detector_line) + + facts = item.get("facts") or {} + if facts: + lines.extend(["", "**Facts:**"]) + lines.extend(f"- {key}: {facts[key]}" for key in sorted(facts)) + + cves = [c for c in item.get("cves") or [] if isinstance(c, dict) and c.get("id")] + if cves: + lines.extend(["", "**CVE verdicts:**"]) + for cve in cves: + cve_line = f"- {cve['id']}: {cve.get('verdict') or 'unknown'}" + if cve.get("reason"): + cve_line += f" ({cve['reason']})" + lines.append(cve_line) + return "\n".join(lines) + + def _unique_id(self, item, target, seen): + # Stable across runs: no line numbers or wording. Identical findings in one + # report are told apart by their order, as the SARIF fingerprint does. + facts = item.get("facts") or {} + parts = [ + f"{item.get('detector')}/{item.get('kind')}", + str(target.get("type") or ""), + str(target.get("target") or ""), + str(item.get("path") or ""), + ] + parts.extend(f"{key}={facts[key]}" for key in IDENTITY_FACTS if facts.get(key)) + key = "\x00".join(parts) + seen[key] = seen.get(key, 0) + 1 + if seen[key] > 1: + key += f"\x00{seen[key]}" + return hashlib.sha256(key.encode("utf-8")).hexdigest() diff --git a/unittests/scans/earlylight/cleared_only.json b/unittests/scans/earlylight/cleared_only.json new file mode 100644 index 00000000000..d43cc85be8e --- /dev/null +++ b/unittests/scans/earlylight/cleared_only.json @@ -0,0 +1,252 @@ +{ + "tool": "earlylight", + "version": "v0.3.2-0.20261103043243-19c2e2404ae6", + "detectors": [ + { + "id": "php2shell", + "title": "PHP2Shell: PHP vulnerable to CVE-2026-109618 and CVE-2026-109619", + "cves": [ + "CVE-2026-109618", + "CVE-2026-109619" + ], + "advisories": [ + "https://github.com/php/php-src/security/advisories/GHSA-w8hw-w9fx-j79w", + "https://github.com/php/php-src/security/advisories/GHSA-6wmf-4rv5-p3rp" + ], + "data": "distribution fix data from 2026-10-21T16:25:30Z" + } + ], + "targets": [ + { + "target": "php:8.3.36-cli", + "type": "image", + "digest": "sha256:d716009046f4acac161ed7a14921111398228c2311696f132686be8613cb19ab", + "platform": "linux/amd64", + "files_scanned": 9113, + "files_read": 1214, + "findings": [ + { + "detector": "php2shell", + "kind": "php-runtime-cli", + "path": "/usr/local/bin/php", + "verdict": "fixed", + "reason": "PHP 8.3.36 is the fixed release", + "message": "PHP 8.3.36 (cli)", + "method": "elf-strings", + "facts": { + "php_version": "8.3.36", + "sapi": "cli" + }, + "cves": [ + { + "id": "CVE-2026-109618", + "verdict": "fixed", + "reason": "PHP 8.3.36 is the fixed release" + }, + { + "id": "CVE-2026-109619", + "verdict": "fixed", + "reason": "PHP 8.3.36 is the fixed release" + } + ], + "severity": "none", + "details": { + "path": "/usr/local/bin/php", + "php_version": "8.3.36", + "sapi": "cli", + "web_facing": false, + "status": "fixed", + "severity": "none", + "reason": "PHP 8.3.36 is the fixed release", + "cves": [ + { + "id": "CVE-2026-109618", + "status": "fixed", + "reason": "PHP 8.3.36 is the fixed release" + }, + { + "id": "CVE-2026-109619", + "status": "fixed", + "reason": "PHP 8.3.36 is the fixed release" + } + ], + "os": { + "id": "debian", + "version_id": "13", + "codename": "trixie", + "pretty_name": "Debian GNU/Linux 13 (trixie)" + } + } + }, + { + "detector": "php2shell", + "kind": "php-runtime-web", + "path": "/usr/local/bin/php-cgi", + "verdict": "fixed", + "reason": "PHP 8.3.36 is the fixed release", + "message": "PHP 8.3.36 (cgi-fcgi)", + "method": "elf-strings", + "facts": { + "php_version": "8.3.36", + "sapi": "cgi-fcgi" + }, + "cves": [ + { + "id": "CVE-2026-109618", + "verdict": "fixed", + "reason": "PHP 8.3.36 is the fixed release" + }, + { + "id": "CVE-2026-109619", + "verdict": "fixed", + "reason": "PHP 8.3.36 is the fixed release" + } + ], + "severity": "none", + "details": { + "path": "/usr/local/bin/php-cgi", + "php_version": "8.3.36", + "sapi": "cgi-fcgi", + "web_facing": true, + "status": "fixed", + "severity": "none", + "reason": "PHP 8.3.36 is the fixed release", + "cves": [ + { + "id": "CVE-2026-109618", + "status": "fixed", + "reason": "PHP 8.3.36 is the fixed release" + }, + { + "id": "CVE-2026-109619", + "status": "fixed", + "reason": "PHP 8.3.36 is the fixed release" + } + ], + "os": { + "id": "debian", + "version_id": "13", + "codename": "trixie", + "pretty_name": "Debian GNU/Linux 13 (trixie)" + } + } + }, + { + "detector": "php2shell", + "kind": "php-runtime-cli", + "path": "/usr/local/bin/phpdbg", + "verdict": "fixed", + "reason": "PHP 8.3.36 is the fixed release", + "message": "PHP 8.3.36 (cli)", + "method": "elf-strings", + "facts": { + "php_version": "8.3.36", + "sapi": "cli" + }, + "cves": [ + { + "id": "CVE-2026-109618", + "verdict": "fixed", + "reason": "PHP 8.3.36 is the fixed release" + }, + { + "id": "CVE-2026-109619", + "verdict": "fixed", + "reason": "PHP 8.3.36 is the fixed release" + } + ], + "severity": "none", + "details": { + "path": "/usr/local/bin/phpdbg", + "php_version": "8.3.36", + "sapi": "cli", + "web_facing": false, + "status": "fixed", + "severity": "none", + "reason": "PHP 8.3.36 is the fixed release", + "cves": [ + { + "id": "CVE-2026-109618", + "status": "fixed", + "reason": "PHP 8.3.36 is the fixed release" + }, + { + "id": "CVE-2026-109619", + "status": "fixed", + "reason": "PHP 8.3.36 is the fixed release" + } + ], + "os": { + "id": "debian", + "version_id": "13", + "codename": "trixie", + "pretty_name": "Debian GNU/Linux 13 (trixie)" + } + } + }, + { + "detector": "php2shell", + "kind": "php-runtime-cli", + "path": "/usr/local/lib/libphp.so", + "verdict": "fixed", + "reason": "PHP 8.3.36 is the fixed release", + "message": "PHP 8.3.36 (embed)", + "method": "elf-strings", + "facts": { + "php_version": "8.3.36", + "sapi": "embed" + }, + "cves": [ + { + "id": "CVE-2026-109618", + "verdict": "fixed", + "reason": "PHP 8.3.36 is the fixed release" + }, + { + "id": "CVE-2026-109619", + "verdict": "fixed", + "reason": "PHP 8.3.36 is the fixed release" + } + ], + "severity": "none", + "details": { + "path": "/usr/local/lib/libphp.so", + "php_version": "8.3.36", + "sapi": "embed", + "web_facing": false, + "status": "fixed", + "severity": "none", + "reason": "PHP 8.3.36 is the fixed release", + "cves": [ + { + "id": "CVE-2026-109618", + "status": "fixed", + "reason": "PHP 8.3.36 is the fixed release" + }, + { + "id": "CVE-2026-109619", + "status": "fixed", + "reason": "PHP 8.3.36 is the fixed release" + } + ], + "os": { + "id": "debian", + "version_id": "13", + "codename": "trixie", + "pretty_name": "Debian GNU/Linux 13 (trixie)" + } + } + } + ], + "skipped": [], + "annotations": { + "os": { + "id": "debian", + "version_id": "13", + "codename": "trixie", + "pretty_name": "Debian GNU/Linux 13 (trixie)" + } + } + } + ] +} diff --git a/unittests/scans/earlylight/many_targets.json b/unittests/scans/earlylight/many_targets.json new file mode 100644 index 00000000000..63e08fc5eea --- /dev/null +++ b/unittests/scans/earlylight/many_targets.json @@ -0,0 +1,546 @@ +{ + "tool": "earlylight", + "version": "v0.3.2-0.20261103043243-19c2e2404ae6", + "detectors": [ + { + "id": "php2shell", + "title": "PHP2Shell: PHP vulnerable to CVE-2026-109618 and CVE-2026-109619", + "cves": [ + "CVE-2026-109618", + "CVE-2026-109619" + ], + "advisories": [ + "https://github.com/php/php-src/security/advisories/GHSA-w8hw-w9fx-j79w", + "https://github.com/php/php-src/security/advisories/GHSA-6wmf-4rv5-p3rp" + ], + "data": "distribution fix data from 2026-10-21T16:25:30Z" + }, + { + "id": "brotli", + "title": "Brotli decoder vulnerable to CVE-2026-111497", + "cves": [ + "CVE-2026-111497" + ], + "advisories": [ + "https://github.com/advisories/GHSA-9v8h-qqgm-4m35", + "https://github.com/google/brotli/releases/tag/v1.2.1", + "https://nodejs.org/en/blog/vulnerability/october-2026-security-releases", + "https://github.com/dotnet/announcements/issues/494" + ], + "data": "Brotli decoder error-code tables (google/brotli 1.0.0 to 1.2.1) and the fixed Node.js, .NET and Bun releases" + } + ], + "targets": [ + { + "target": "php:8.3.36-cli", + "type": "image", + "digest": "sha256:d716009046f4acac161ed7a14921111398228c2311696f132686be8613cb19ab", + "platform": "linux/amd64", + "files_scanned": 9113, + "files_read": 1251, + "findings": [ + { + "detector": "php2shell", + "kind": "php-runtime-cli", + "path": "/usr/local/bin/php", + "verdict": "fixed", + "reason": "PHP 8.3.36 is the fixed release", + "message": "PHP 8.3.36 (cli)", + "method": "elf-strings", + "facts": { + "php_version": "8.3.36", + "sapi": "cli" + }, + "cves": [ + { + "id": "CVE-2026-109618", + "verdict": "fixed", + "reason": "PHP 8.3.36 is the fixed release" + }, + { + "id": "CVE-2026-109619", + "verdict": "fixed", + "reason": "PHP 8.3.36 is the fixed release" + } + ], + "severity": "none", + "details": { + "path": "/usr/local/bin/php", + "php_version": "8.3.36", + "sapi": "cli", + "web_facing": false, + "status": "fixed", + "severity": "none", + "reason": "PHP 8.3.36 is the fixed release", + "cves": [ + { + "id": "CVE-2026-109618", + "status": "fixed", + "reason": "PHP 8.3.36 is the fixed release" + }, + { + "id": "CVE-2026-109619", + "status": "fixed", + "reason": "PHP 8.3.36 is the fixed release" + } + ], + "os": { + "id": "debian", + "version_id": "13", + "codename": "trixie", + "pretty_name": "Debian GNU/Linux 13 (trixie)" + } + } + }, + { + "detector": "php2shell", + "kind": "php-runtime-web", + "path": "/usr/local/bin/php-cgi", + "verdict": "fixed", + "reason": "PHP 8.3.36 is the fixed release", + "message": "PHP 8.3.36 (cgi-fcgi)", + "method": "elf-strings", + "facts": { + "php_version": "8.3.36", + "sapi": "cgi-fcgi" + }, + "cves": [ + { + "id": "CVE-2026-109618", + "verdict": "fixed", + "reason": "PHP 8.3.36 is the fixed release" + }, + { + "id": "CVE-2026-109619", + "verdict": "fixed", + "reason": "PHP 8.3.36 is the fixed release" + } + ], + "severity": "none", + "details": { + "path": "/usr/local/bin/php-cgi", + "php_version": "8.3.36", + "sapi": "cgi-fcgi", + "web_facing": true, + "status": "fixed", + "severity": "none", + "reason": "PHP 8.3.36 is the fixed release", + "cves": [ + { + "id": "CVE-2026-109618", + "status": "fixed", + "reason": "PHP 8.3.36 is the fixed release" + }, + { + "id": "CVE-2026-109619", + "status": "fixed", + "reason": "PHP 8.3.36 is the fixed release" + } + ], + "os": { + "id": "debian", + "version_id": "13", + "codename": "trixie", + "pretty_name": "Debian GNU/Linux 13 (trixie)" + } + } + }, + { + "detector": "php2shell", + "kind": "php-runtime-cli", + "path": "/usr/local/bin/phpdbg", + "verdict": "fixed", + "reason": "PHP 8.3.36 is the fixed release", + "message": "PHP 8.3.36 (cli)", + "method": "elf-strings", + "facts": { + "php_version": "8.3.36", + "sapi": "cli" + }, + "cves": [ + { + "id": "CVE-2026-109618", + "verdict": "fixed", + "reason": "PHP 8.3.36 is the fixed release" + }, + { + "id": "CVE-2026-109619", + "verdict": "fixed", + "reason": "PHP 8.3.36 is the fixed release" + } + ], + "severity": "none", + "details": { + "path": "/usr/local/bin/phpdbg", + "php_version": "8.3.36", + "sapi": "cli", + "web_facing": false, + "status": "fixed", + "severity": "none", + "reason": "PHP 8.3.36 is the fixed release", + "cves": [ + { + "id": "CVE-2026-109618", + "status": "fixed", + "reason": "PHP 8.3.36 is the fixed release" + }, + { + "id": "CVE-2026-109619", + "status": "fixed", + "reason": "PHP 8.3.36 is the fixed release" + } + ], + "os": { + "id": "debian", + "version_id": "13", + "codename": "trixie", + "pretty_name": "Debian GNU/Linux 13 (trixie)" + } + } + }, + { + "detector": "php2shell", + "kind": "php-runtime-cli", + "path": "/usr/local/lib/libphp.so", + "verdict": "fixed", + "reason": "PHP 8.3.36 is the fixed release", + "message": "PHP 8.3.36 (embed)", + "method": "elf-strings", + "facts": { + "php_version": "8.3.36", + "sapi": "embed" + }, + "cves": [ + { + "id": "CVE-2026-109618", + "verdict": "fixed", + "reason": "PHP 8.3.36 is the fixed release" + }, + { + "id": "CVE-2026-109619", + "verdict": "fixed", + "reason": "PHP 8.3.36 is the fixed release" + } + ], + "severity": "none", + "details": { + "path": "/usr/local/lib/libphp.so", + "php_version": "8.3.36", + "sapi": "embed", + "web_facing": false, + "status": "fixed", + "severity": "none", + "reason": "PHP 8.3.36 is the fixed release", + "cves": [ + { + "id": "CVE-2026-109618", + "status": "fixed", + "reason": "PHP 8.3.36 is the fixed release" + }, + { + "id": "CVE-2026-109619", + "status": "fixed", + "reason": "PHP 8.3.36 is the fixed release" + } + ], + "os": { + "id": "debian", + "version_id": "13", + "codename": "trixie", + "pretty_name": "Debian GNU/Linux 13 (trixie)" + } + } + }, + { + "detector": "brotli", + "kind": "decoder-library", + "path": "/usr/lib/x86_64-linux-gnu/libbrotlidec.so.1.1.0", + "verdict": "fixed", + "reason": "the decoder's error table has BLOCK_SWITCH, the error code the fix added, so it has the fix (upstream or a vendor backport)", + "message": "Brotli decoder with the CVE-2026-111497 fix (libbrotlidec)", + "method": "brotli-error-table", + "facts": { + "block_switch": "yes", + "distance": "yes", + "error_table": "prefixed", + "product": "libbrotlidec" + }, + "cves": [ + { + "id": "CVE-2026-111497", + "verdict": "fixed", + "reason": "the decoder's error table has BLOCK_SWITCH, the error code the fix added, so it has the fix (upstream or a vendor backport)" + } + ], + "details": { + "kind": "decoder-library", + "path": "/usr/lib/x86_64-linux-gnu/libbrotlidec.so.1.1.0", + "status": "fixed", + "reason": "the decoder's error table has BLOCK_SWITCH, the error code the fix added, so it has the fix (upstream or a vendor backport)", + "message": "Brotli decoder with the CVE-2026-111497 fix (libbrotlidec)", + "method": "brotli-error-table", + "product": "libbrotlidec", + "table": "prefixed", + "block_switch": true, + "distance": true + } + } + ], + "skipped": [ + { + "path": "/usr/src/php.tar.xz!/php-8.3.36/ext/bz2/tests/004_2.txt.bz2", + "reason": "bzip2: bzip2 data invalid: block checksum mismatch" + }, + { + "path": "/usr/src/php.tar.xz!/php-8.3.36/ext/bz2/tests/72613.bz2", + "reason": "bzip2: bzip2 data invalid: block checksum mismatch" + } + ], + "annotations": { + "os": { + "id": "debian", + "version_id": "13", + "codename": "trixie", + "pretty_name": "Debian GNU/Linux 13 (trixie)" + } + } + }, + { + "target": "trafex/php-nginx:2.0.1", + "type": "image", + "digest": "sha256:c9c6d912ce5dbb57e204e7368cacad02530e56f068a8ba10013e0a8cda66983e", + "platform": "linux/amd64", + "files_scanned": 3734, + "files_read": 200, + "findings": [ + { + "detector": "php2shell", + "kind": "php-runtime-cli", + "path": "/usr/bin/php8", + "verdict": "unknown", + "reason": "installed by php8 8.0.2-r0 (apk package php8) on Alpine Linux v3.13; this tool has no fix data for that package stream, and the vendor may have backported the fix without changing the version PHP reports", + "message": "PHP 8.0.2 (cli)", + "method": "elf-strings", + "facts": { + "package": "php8 8.0.2-r0", + "php_version": "8.0.2", + "sapi": "cli" + }, + "cves": [ + { + "id": "CVE-2026-109618", + "verdict": "unknown", + "reason": "installed by php8 8.0.2-r0 (apk package php8) on Alpine Linux v3.13; this tool has no fix data for that package stream, and the vendor may have backported the fix without changing the version PHP reports" + }, + { + "id": "CVE-2026-109619", + "verdict": "unknown", + "reason": "installed by php8 8.0.2-r0 (apk package php8) on Alpine Linux v3.13; this tool has no fix data for that package stream, and the vendor may have backported the fix without changing the version PHP reports" + } + ], + "fix": "Check the security advisories for php8 on Alpine Linux v3.13 and upgrade the package if a fix is listed.", + "severity": "review", + "details": { + "path": "/usr/bin/php8", + "php_version": "8.0.2", + "sapi": "cli", + "web_facing": false, + "status": "unknown", + "severity": "review", + "reason": "installed by php8 8.0.2-r0 (apk package php8) on Alpine Linux v3.13; this tool has no fix data for that package stream, and the vendor may have backported the fix without changing the version PHP reports", + "cves": [ + { + "id": "CVE-2026-109618", + "status": "unknown", + "reason": "installed by php8 8.0.2-r0 (apk package php8) on Alpine Linux v3.13; this tool has no fix data for that package stream, and the vendor may have backported the fix without changing the version PHP reports" + }, + { + "id": "CVE-2026-109619", + "status": "unknown", + "reason": "installed by php8 8.0.2-r0 (apk package php8) on Alpine Linux v3.13; this tool has no fix data for that package stream, and the vendor may have backported the fix without changing the version PHP reports" + } + ], + "package": { + "format": "apk", + "name": "php8", + "version": "8.0.2-r0", + "source": "php8", + "arch": "x86_64", + "db": "/lib/apk/db/installed" + }, + "os": { + "id": "alpine", + "version_id": "3.13.5", + "pretty_name": "Alpine Linux v3.13" + }, + "fix": "Check the security advisories for php8 on Alpine Linux v3.13 and upgrade the package if a fix is listed." + } + }, + { + "detector": "php2shell", + "kind": "php-runtime-web", + "path": "/usr/sbin/php-fpm8", + "verdict": "unknown", + "reason": "installed by php8-fpm 8.0.2-r0 (apk package php8) on Alpine Linux v3.13; this tool has no fix data for that package stream, and the vendor may have backported the fix without changing the version PHP reports", + "message": "PHP 8.0.2 (fpm-fcgi)", + "method": "elf-strings", + "facts": { + "package": "php8-fpm 8.0.2-r0", + "php_version": "8.0.2", + "sapi": "fpm-fcgi" + }, + "cves": [ + { + "id": "CVE-2026-109618", + "verdict": "unknown", + "reason": "installed by php8-fpm 8.0.2-r0 (apk package php8) on Alpine Linux v3.13; this tool has no fix data for that package stream, and the vendor may have backported the fix without changing the version PHP reports" + }, + { + "id": "CVE-2026-109619", + "verdict": "unknown", + "reason": "installed by php8-fpm 8.0.2-r0 (apk package php8) on Alpine Linux v3.13; this tool has no fix data for that package stream, and the vendor may have backported the fix without changing the version PHP reports" + } + ], + "fix": "Check the security advisories for php8 on Alpine Linux v3.13 and upgrade the package if a fix is listed.", + "severity": "review", + "details": { + "path": "/usr/sbin/php-fpm8", + "php_version": "8.0.2", + "sapi": "fpm-fcgi", + "web_facing": true, + "status": "unknown", + "severity": "review", + "reason": "installed by php8-fpm 8.0.2-r0 (apk package php8) on Alpine Linux v3.13; this tool has no fix data for that package stream, and the vendor may have backported the fix without changing the version PHP reports", + "cves": [ + { + "id": "CVE-2026-109618", + "status": "unknown", + "reason": "installed by php8-fpm 8.0.2-r0 (apk package php8) on Alpine Linux v3.13; this tool has no fix data for that package stream, and the vendor may have backported the fix without changing the version PHP reports" + }, + { + "id": "CVE-2026-109619", + "status": "unknown", + "reason": "installed by php8-fpm 8.0.2-r0 (apk package php8) on Alpine Linux v3.13; this tool has no fix data for that package stream, and the vendor may have backported the fix without changing the version PHP reports" + } + ], + "package": { + "format": "apk", + "name": "php8-fpm", + "version": "8.0.2-r0", + "source": "php8", + "arch": "x86_64", + "db": "/lib/apk/db/installed" + }, + "os": { + "id": "alpine", + "version_id": "3.13.5", + "pretty_name": "Alpine Linux v3.13" + }, + "fix": "Check the security advisories for php8 on Alpine Linux v3.13 and upgrade the package if a fix is listed." + } + }, + { + "detector": "brotli", + "kind": "decoder-library", + "path": "/usr/lib/libbrotlidec.so.1.0.9", + "verdict": "vulnerable", + "reason": "the decoder's error table has DISTANCE (Brotli 1.0.3 and later) but not BLOCK_SWITCH, the error code the fix added", + "message": "Brotli decoder without the CVE-2026-111497 fix (libbrotlidec)", + "method": "brotli-error-table", + "facts": { + "block_switch": "no", + "distance": "yes", + "error_table": "bare", + "product": "libbrotlidec" + }, + "cves": [ + { + "id": "CVE-2026-111497", + "verdict": "vulnerable", + "reason": "the decoder's error table has DISTANCE (Brotli 1.0.3 and later) but not BLOCK_SWITCH, the error code the fix added" + } + ], + "fix": "Update the system Brotli package (libbrotli1, brotli-libs, libbrotli or brotli) to your vendor's fixed build, or build Brotli 1.0.10, 1.1.1 or 1.2.1, then restart the programs that load it.", + "details": { + "kind": "decoder-library", + "path": "/usr/lib/libbrotlidec.so.1.0.9", + "status": "vulnerable", + "reason": "the decoder's error table has DISTANCE (Brotli 1.0.3 and later) but not BLOCK_SWITCH, the error code the fix added", + "message": "Brotli decoder without the CVE-2026-111497 fix (libbrotlidec)", + "method": "brotli-error-table", + "product": "libbrotlidec", + "fix": "Update the system Brotli package (libbrotli1, brotli-libs, libbrotli or brotli) to your vendor's fixed build, or build Brotli 1.0.10, 1.1.1 or 1.2.1, then restart the programs that load it.", + "table": "bare", + "block_switch": false, + "distance": true + } + } + ], + "skipped": [], + "annotations": { + "os": { + "id": "alpine", + "version_id": "3.13.5", + "pretty_name": "Alpine Linux v3.13" + } + } + }, + { + "target": "node:20.0.0-slim", + "type": "image", + "digest": "sha256:130e0957d64695f37eea9ff80541f628e811da1d969c36988759738e39a0ab3b", + "platform": "linux/amd64", + "files_scanned": 8066, + "files_read": 900, + "findings": [ + { + "detector": "brotli", + "kind": "bundled-decoder", + "path": "/usr/local/bin/node", + "verdict": "vulnerable", + "reason": "the decoder's error table has DISTANCE (Brotli 1.0.3 and later) but not BLOCK_SWITCH, the error code the fix added", + "message": "Brotli decoder without the CVE-2026-111497 fix (Node.js 20.0.0)", + "method": "brotli-error-table", + "facts": { + "block_switch": "no", + "distance": "yes", + "error_table": "bare", + "product": "node", + "version": "20.0.0" + }, + "cves": [ + { + "id": "CVE-2026-111497", + "verdict": "vulnerable", + "reason": "the decoder's error table has DISTANCE (Brotli 1.0.3 and later) but not BLOCK_SWITCH, the error code the fix added" + } + ], + "fix": "Node.js 20 has no fixed release: move to 22.23.4, 24.22.1 or 26.12.1.", + "details": { + "kind": "bundled-decoder", + "path": "/usr/local/bin/node", + "status": "vulnerable", + "reason": "the decoder's error table has DISTANCE (Brotli 1.0.3 and later) but not BLOCK_SWITCH, the error code the fix added", + "message": "Brotli decoder without the CVE-2026-111497 fix (Node.js 20.0.0)", + "method": "brotli-error-table", + "product": "node", + "version": "20.0.0", + "fix": "Node.js 20 has no fixed release: move to 22.23.4, 24.22.1 or 26.12.1.", + "table": "bare", + "block_switch": false, + "distance": true + } + } + ], + "skipped": [], + "annotations": { + "os": { + "id": "debian", + "version_id": "11", + "codename": "bullseye", + "pretty_name": "Debian GNU/Linux 11 (bullseye)" + } + } + } + ] +} diff --git a/unittests/scans/earlylight/many_vulns.json b/unittests/scans/earlylight/many_vulns.json new file mode 100644 index 00000000000..f8b2ed2bc04 --- /dev/null +++ b/unittests/scans/earlylight/many_vulns.json @@ -0,0 +1,264 @@ +{ + "tool": "earlylight", + "version": "v0.3.2-0.20261103043243-19c2e2404ae6", + "detectors": [ + { + "id": "brotli", + "title": "Brotli decoder vulnerable to CVE-2026-111497", + "cves": [ + "CVE-2026-111497" + ], + "advisories": [ + "https://github.com/advisories/GHSA-9v8h-qqgm-4m35", + "https://github.com/google/brotli/releases/tag/v1.2.1", + "https://nodejs.org/en/blog/vulnerability/october-2026-security-releases", + "https://github.com/dotnet/announcements/issues/494" + ], + "data": "Brotli decoder error-code tables (google/brotli 1.0.0 to 1.2.1) and the fixed Node.js, .NET and Bun releases" + }, + { + "id": "emudhra", + "title": "Trust in the compromised eMudhra intermediate CA keys", + "cves": [ + "CVE-2026-111943" + ], + "advisories": [ + "https://bugzilla.mozilla.org/show_bug.cgi?id=2083449" + ], + "data": "eight compromised intermediates and four path roots (Mozilla bug 2083449)" + }, + { + "id": "php2shell", + "title": "PHP2Shell: PHP vulnerable to CVE-2026-109618 and CVE-2026-109619", + "cves": [ + "CVE-2026-109618", + "CVE-2026-109619" + ], + "advisories": [ + "https://github.com/php/php-src/security/advisories/GHSA-w8hw-w9fx-j79w", + "https://github.com/php/php-src/security/advisories/GHSA-6wmf-4rv5-p3rp" + ], + "data": "distribution fix data from 2026-10-21T16:25:30Z" + } + ], + "targets": [ + { + "target": "php:8.3.0-cli", + "type": "image", + "digest": "sha256:33b1f2a2edaa92c73b41cbb078e8c1abce9b4ebe41af870415c6361830667da7", + "platform": "linux/amd64", + "files_scanned": 8854, + "files_read": 8854, + "findings": [ + { + "detector": "brotli", + "kind": "decoder-library", + "path": "/usr/lib/x86_64-linux-gnu/libbrotlidec.so.1.0.9", + "verdict": "vulnerable", + "reason": "the decoder's error table has DISTANCE (Brotli 1.0.3 and later) but not BLOCK_SWITCH, the error code the fix added", + "message": "Brotli decoder without the CVE-2026-111497 fix (libbrotlidec)", + "method": "brotli-error-table", + "facts": { + "block_switch": "no", + "distance": "yes", + "error_table": "bare", + "product": "libbrotlidec" + }, + "cves": [ + { + "id": "CVE-2026-111497", + "verdict": "vulnerable", + "reason": "the decoder's error table has DISTANCE (Brotli 1.0.3 and later) but not BLOCK_SWITCH, the error code the fix added" + } + ], + "fix": "Update the system Brotli package (libbrotli1, brotli-libs, libbrotli or brotli) to your vendor's fixed build, or build Brotli 1.0.10, 1.1.1 or 1.2.1, then restart the programs that load it.", + "details": { + "kind": "decoder-library", + "path": "/usr/lib/x86_64-linux-gnu/libbrotlidec.so.1.0.9", + "status": "vulnerable", + "reason": "the decoder's error table has DISTANCE (Brotli 1.0.3 and later) but not BLOCK_SWITCH, the error code the fix added", + "message": "Brotli decoder without the CVE-2026-111497 fix (libbrotlidec)", + "method": "brotli-error-table", + "product": "libbrotlidec", + "fix": "Update the system Brotli package (libbrotli1, brotli-libs, libbrotli or brotli) to your vendor's fixed build, or build Brotli 1.0.10, 1.1.1 or 1.2.1, then restart the programs that load it.", + "table": "bare", + "block_switch": false, + "distance": true + } + }, + { + "detector": "emudhra", + "kind": "path-root", + "path": "/etc/ssl/certs/ca-certificates.crt", + "verdict": "vulnerable", + "message": "contains CA \"AAA Certificate Services\", which issued or cross-signed the compromised eMudhra intermediates", + "method": "pem", + "facts": { + "cert_sha256": "D7A7A0FB5D7E2731D771E9484EBCDEF71D5F0C3E0A2948782BC83EE0EA699EF4", + "spki_sha256": "BD153ED7B0434F6886B17BCE8BBE84ED340C7132D702A8F4FA318F756ECBD6F3", + "subject": "AAA Certificate Services" + }, + "details": { + "kind": "path-root", + "path": "/etc/ssl/certs/ca-certificates.crt", + "subject": "AAA Certificate Services", + "spki_sha256": "BD153ED7B0434F6886B17BCE8BBE84ED340C7132D702A8F4FA318F756ECBD6F3", + "cert_sha256": "D7A7A0FB5D7E2731D771E9484EBCDEF71D5F0C3E0A2948782BC83EE0EA699EF4", + "method": "pem", + "detail": "contains CA \"AAA Certificate Services\", which issued or cross-signed the compromised eMudhra intermediates" + } + }, + { + "detector": "emudhra", + "kind": "path-root", + "path": "/etc/ssl/certs/ca-certificates.crt", + "verdict": "vulnerable", + "message": "contains CA \"emSign ECC Root CA - G3\", which issued or cross-signed the compromised eMudhra intermediates", + "method": "pem", + "facts": { + "cert_sha256": "86A1ECBA089C4A8D3BBE2734C612BA341D813E043CF9E8A862CD5C57A36BBE6B", + "spki_sha256": "8D417DB2DD8BF5E3084D1E3F196D583849D81BDD4C00C70B9D39369E96B8C782", + "subject": "emSign ECC Root CA - G3" + }, + "details": { + "kind": "path-root", + "path": "/etc/ssl/certs/ca-certificates.crt", + "subject": "emSign ECC Root CA - G3", + "spki_sha256": "8D417DB2DD8BF5E3084D1E3F196D583849D81BDD4C00C70B9D39369E96B8C782", + "cert_sha256": "86A1ECBA089C4A8D3BBE2734C612BA341D813E043CF9E8A862CD5C57A36BBE6B", + "method": "pem", + "detail": "contains CA \"emSign ECC Root CA - G3\", which issued or cross-signed the compromised eMudhra intermediates" + } + }, + { + "detector": "php2shell", + "kind": "php-runtime-cli", + "path": "/usr/local/bin/php", + "verdict": "vulnerable", + "reason": "PHP 8.3.0 is below the fixed release 8.3.36; no package manager installed this file, so no vendor backport applies", + "message": "PHP 8.3.0 (cli)", + "method": "elf-strings", + "facts": { + "php_version": "8.3.0", + "sapi": "cli" + }, + "cves": [ + { + "id": "CVE-2026-109618", + "verdict": "vulnerable", + "reason": "PHP 8.3.0 is below the fixed release 8.3.36" + }, + { + "id": "CVE-2026-109619", + "verdict": "vulnerable", + "reason": "PHP 8.3.0 is below the fixed release 8.3.36" + } + ], + "fix": "Rebuild on PHP 8.3.36 or later (for the official Docker image, a php:8.3.36 tag), then redeploy.", + "severity": "high", + "details": { + "path": "/usr/local/bin/php", + "php_version": "8.3.0", + "sapi": "cli", + "web_facing": false, + "status": "vulnerable", + "severity": "high", + "reason": "PHP 8.3.0 is below the fixed release 8.3.36; no package manager installed this file, so no vendor backport applies", + "cves": [ + { + "id": "CVE-2026-109618", + "status": "vulnerable", + "reason": "PHP 8.3.0 is below the fixed release 8.3.36" + }, + { + "id": "CVE-2026-109619", + "status": "vulnerable", + "reason": "PHP 8.3.0 is below the fixed release 8.3.36" + } + ], + "os": { + "id": "debian", + "version_id": "12", + "codename": "bookworm", + "pretty_name": "Debian GNU/Linux 12 (bookworm)" + }, + "fix": "Rebuild on PHP 8.3.36 or later (for the official Docker image, a php:8.3.36 tag), then redeploy." + } + }, + { + "detector": "php2shell", + "kind": "php-runtime-web", + "path": "/usr/local/bin/php-cgi", + "verdict": "vulnerable", + "reason": "PHP 8.3.0 is below the fixed release 8.3.36; no package manager installed this file, so no vendor backport applies", + "message": "PHP 8.3.0 (cgi-fcgi)", + "method": "elf-strings", + "facts": { + "php_version": "8.3.0", + "sapi": "cgi-fcgi" + }, + "cves": [ + { + "id": "CVE-2026-109618", + "verdict": "vulnerable", + "reason": "PHP 8.3.0 is below the fixed release 8.3.36" + }, + { + "id": "CVE-2026-109619", + "verdict": "vulnerable", + "reason": "PHP 8.3.0 is below the fixed release 8.3.36" + } + ], + "fix": "Rebuild on PHP 8.3.36 or later (for the official Docker image, a php:8.3.36 tag), then redeploy.", + "severity": "critical", + "details": { + "path": "/usr/local/bin/php-cgi", + "php_version": "8.3.0", + "sapi": "cgi-fcgi", + "web_facing": true, + "status": "vulnerable", + "severity": "critical", + "reason": "PHP 8.3.0 is below the fixed release 8.3.36; no package manager installed this file, so no vendor backport applies", + "cves": [ + { + "id": "CVE-2026-109618", + "status": "vulnerable", + "reason": "PHP 8.3.0 is below the fixed release 8.3.36" + }, + { + "id": "CVE-2026-109619", + "status": "vulnerable", + "reason": "PHP 8.3.0 is below the fixed release 8.3.36" + } + ], + "os": { + "id": "debian", + "version_id": "12", + "codename": "bookworm", + "pretty_name": "Debian GNU/Linux 12 (bookworm)" + }, + "fix": "Rebuild on PHP 8.3.36 or later (for the official Docker image, a php:8.3.36 tag), then redeploy." + } + } + ], + "skipped": [ + { + "path": "/usr/src/php.tar.xz!/php-8.3.0/ext/bz2/tests/004_2.txt.bz2", + "reason": "bzip2: bzip2 data invalid: block checksum mismatch" + }, + { + "path": "/usr/src/php.tar.xz!/php-8.3.0/ext/bz2/tests/72613.bz2", + "reason": "bzip2: bzip2 data invalid: block checksum mismatch" + } + ], + "annotations": { + "os": { + "id": "debian", + "version_id": "12", + "codename": "bookworm", + "pretty_name": "Debian GNU/Linux 12 (bookworm)" + } + } + } + ] +} diff --git a/unittests/scans/earlylight/needs_review.json b/unittests/scans/earlylight/needs_review.json new file mode 100644 index 00000000000..9c9ee90e7ef --- /dev/null +++ b/unittests/scans/earlylight/needs_review.json @@ -0,0 +1,165 @@ +{ + "tool": "earlylight", + "version": "v0.3.2-0.20261103043243-19c2e2404ae6", + "detectors": [ + { + "id": "php2shell", + "title": "PHP2Shell: PHP vulnerable to CVE-2026-109618 and CVE-2026-109619", + "cves": [ + "CVE-2026-109618", + "CVE-2026-109619" + ], + "advisories": [ + "https://github.com/php/php-src/security/advisories/GHSA-w8hw-w9fx-j79w", + "https://github.com/php/php-src/security/advisories/GHSA-6wmf-4rv5-p3rp" + ], + "data": "distribution fix data from 2026-10-21T16:25:30Z" + } + ], + "targets": [ + { + "target": "trafex/php-nginx:2.0.1", + "type": "image", + "digest": "sha256:c9c6d912ce5dbb57e204e7368cacad02530e56f068a8ba10013e0a8cda66983e", + "platform": "linux/amd64", + "files_scanned": 3734, + "files_read": 197, + "findings": [ + { + "detector": "php2shell", + "kind": "php-runtime-cli", + "path": "/usr/bin/php8", + "verdict": "unknown", + "reason": "installed by php8 8.0.2-r0 (apk package php8) on Alpine Linux v3.13; this tool has no fix data for that package stream, and the vendor may have backported the fix without changing the version PHP reports", + "message": "PHP 8.0.2 (cli)", + "method": "elf-strings", + "facts": { + "package": "php8 8.0.2-r0", + "php_version": "8.0.2", + "sapi": "cli" + }, + "cves": [ + { + "id": "CVE-2026-109618", + "verdict": "unknown", + "reason": "installed by php8 8.0.2-r0 (apk package php8) on Alpine Linux v3.13; this tool has no fix data for that package stream, and the vendor may have backported the fix without changing the version PHP reports" + }, + { + "id": "CVE-2026-109619", + "verdict": "unknown", + "reason": "installed by php8 8.0.2-r0 (apk package php8) on Alpine Linux v3.13; this tool has no fix data for that package stream, and the vendor may have backported the fix without changing the version PHP reports" + } + ], + "fix": "Check the security advisories for php8 on Alpine Linux v3.13 and upgrade the package if a fix is listed.", + "severity": "review", + "details": { + "path": "/usr/bin/php8", + "php_version": "8.0.2", + "sapi": "cli", + "web_facing": false, + "status": "unknown", + "severity": "review", + "reason": "installed by php8 8.0.2-r0 (apk package php8) on Alpine Linux v3.13; this tool has no fix data for that package stream, and the vendor may have backported the fix without changing the version PHP reports", + "cves": [ + { + "id": "CVE-2026-109618", + "status": "unknown", + "reason": "installed by php8 8.0.2-r0 (apk package php8) on Alpine Linux v3.13; this tool has no fix data for that package stream, and the vendor may have backported the fix without changing the version PHP reports" + }, + { + "id": "CVE-2026-109619", + "status": "unknown", + "reason": "installed by php8 8.0.2-r0 (apk package php8) on Alpine Linux v3.13; this tool has no fix data for that package stream, and the vendor may have backported the fix without changing the version PHP reports" + } + ], + "package": { + "format": "apk", + "name": "php8", + "version": "8.0.2-r0", + "source": "php8", + "arch": "x86_64", + "db": "/lib/apk/db/installed" + }, + "os": { + "id": "alpine", + "version_id": "3.13.5", + "pretty_name": "Alpine Linux v3.13" + }, + "fix": "Check the security advisories for php8 on Alpine Linux v3.13 and upgrade the package if a fix is listed." + } + }, + { + "detector": "php2shell", + "kind": "php-runtime-web", + "path": "/usr/sbin/php-fpm8", + "verdict": "unknown", + "reason": "installed by php8-fpm 8.0.2-r0 (apk package php8) on Alpine Linux v3.13; this tool has no fix data for that package stream, and the vendor may have backported the fix without changing the version PHP reports", + "message": "PHP 8.0.2 (fpm-fcgi)", + "method": "elf-strings", + "facts": { + "package": "php8-fpm 8.0.2-r0", + "php_version": "8.0.2", + "sapi": "fpm-fcgi" + }, + "cves": [ + { + "id": "CVE-2026-109618", + "verdict": "unknown", + "reason": "installed by php8-fpm 8.0.2-r0 (apk package php8) on Alpine Linux v3.13; this tool has no fix data for that package stream, and the vendor may have backported the fix without changing the version PHP reports" + }, + { + "id": "CVE-2026-109619", + "verdict": "unknown", + "reason": "installed by php8-fpm 8.0.2-r0 (apk package php8) on Alpine Linux v3.13; this tool has no fix data for that package stream, and the vendor may have backported the fix without changing the version PHP reports" + } + ], + "fix": "Check the security advisories for php8 on Alpine Linux v3.13 and upgrade the package if a fix is listed.", + "severity": "review", + "details": { + "path": "/usr/sbin/php-fpm8", + "php_version": "8.0.2", + "sapi": "fpm-fcgi", + "web_facing": true, + "status": "unknown", + "severity": "review", + "reason": "installed by php8-fpm 8.0.2-r0 (apk package php8) on Alpine Linux v3.13; this tool has no fix data for that package stream, and the vendor may have backported the fix without changing the version PHP reports", + "cves": [ + { + "id": "CVE-2026-109618", + "status": "unknown", + "reason": "installed by php8-fpm 8.0.2-r0 (apk package php8) on Alpine Linux v3.13; this tool has no fix data for that package stream, and the vendor may have backported the fix without changing the version PHP reports" + }, + { + "id": "CVE-2026-109619", + "status": "unknown", + "reason": "installed by php8-fpm 8.0.2-r0 (apk package php8) on Alpine Linux v3.13; this tool has no fix data for that package stream, and the vendor may have backported the fix without changing the version PHP reports" + } + ], + "package": { + "format": "apk", + "name": "php8-fpm", + "version": "8.0.2-r0", + "source": "php8", + "arch": "x86_64", + "db": "/lib/apk/db/installed" + }, + "os": { + "id": "alpine", + "version_id": "3.13.5", + "pretty_name": "Alpine Linux v3.13" + }, + "fix": "Check the security advisories for php8 on Alpine Linux v3.13 and upgrade the package if a fix is listed." + } + } + ], + "skipped": [], + "annotations": { + "os": { + "id": "alpine", + "version_id": "3.13.5", + "pretty_name": "Alpine Linux v3.13" + } + } + } + ] +} diff --git a/unittests/scans/earlylight/no_findings.json b/unittests/scans/earlylight/no_findings.json new file mode 100644 index 00000000000..ffad241d8b0 --- /dev/null +++ b/unittests/scans/earlylight/no_findings.json @@ -0,0 +1,55 @@ +{ + "tool": "earlylight", + "version": "v0.3.2-0.20261103043243-19c2e2404ae6", + "detectors": [ + { + "id": "brotli", + "title": "Brotli decoder vulnerable to CVE-2026-111497", + "cves": [ + "CVE-2026-111497" + ], + "advisories": [ + "https://github.com/advisories/GHSA-9v8h-qqgm-4m35", + "https://github.com/google/brotli/releases/tag/v1.2.1", + "https://nodejs.org/en/blog/vulnerability/october-2026-security-releases", + "https://github.com/dotnet/announcements/issues/494" + ], + "data": "Brotli decoder error-code tables (google/brotli 1.0.0 to 1.2.1) and the fixed Node.js, .NET and Bun releases" + }, + { + "id": "emudhra", + "title": "Trust in the compromised eMudhra intermediate CA keys", + "cves": [ + "CVE-2026-111943" + ], + "advisories": [ + "https://bugzilla.mozilla.org/show_bug.cgi?id=2083449" + ], + "data": "eight compromised intermediates and four path roots (Mozilla bug 2083449)" + }, + { + "id": "php2shell", + "title": "PHP2Shell: PHP vulnerable to CVE-2026-109618 and CVE-2026-109619", + "cves": [ + "CVE-2026-109618", + "CVE-2026-109619" + ], + "advisories": [ + "https://github.com/php/php-src/security/advisories/GHSA-w8hw-w9fx-j79w", + "https://github.com/php/php-src/security/advisories/GHSA-6wmf-4rv5-p3rp" + ], + "data": "distribution fix data from 2026-10-21T16:25:30Z" + } + ], + "targets": [ + { + "target": "/srv/app", + "type": "filesystem", + "root": "/srv/app", + "files_scanned": 1, + "files_read": 1, + "findings": [], + "skipped": [] + } + ] +} diff --git a/unittests/scans/earlylight/one_vuln.json b/unittests/scans/earlylight/one_vuln.json new file mode 100644 index 00000000000..829e46abc76 --- /dev/null +++ b/unittests/scans/earlylight/one_vuln.json @@ -0,0 +1,129 @@ +{ + "tool": "earlylight", + "version": "v0.3.2-0.20261103043243-19c2e2404ae6", + "detectors": [ + { + "id": "brotli", + "title": "Brotli decoder vulnerable to CVE-2026-111497", + "cves": [ + "CVE-2026-111497" + ], + "advisories": [ + "https://github.com/advisories/GHSA-9v8h-qqgm-4m35", + "https://github.com/google/brotli/releases/tag/v1.2.1", + "https://nodejs.org/en/blog/vulnerability/october-2026-security-releases", + "https://github.com/dotnet/announcements/issues/494" + ], + "data": "Brotli decoder error-code tables (google/brotli 1.0.0 to 1.2.1) and the fixed Node.js, .NET and Bun releases" + }, + { + "id": "emudhra", + "title": "Trust in the compromised eMudhra intermediate CA keys", + "cves": [ + "CVE-2026-111943" + ], + "advisories": [ + "https://bugzilla.mozilla.org/show_bug.cgi?id=2083449" + ], + "data": "eight compromised intermediates and four path roots (Mozilla bug 2083449)" + }, + { + "id": "php2shell", + "title": "PHP2Shell: PHP vulnerable to CVE-2026-109618 and CVE-2026-109619", + "cves": [ + "CVE-2026-109618", + "CVE-2026-109619" + ], + "advisories": [ + "https://github.com/php/php-src/security/advisories/GHSA-w8hw-w9fx-j79w", + "https://github.com/php/php-src/security/advisories/GHSA-6wmf-4rv5-p3rp" + ], + "data": "distribution fix data from 2026-10-21T16:25:30Z" + } + ], + "targets": [ + { + "target": "php:8.3.0-cli", + "type": "image", + "digest": "sha256:33b1f2a2edaa92c73b41cbb078e8c1abce9b4ebe41af870415c6361830667da7", + "platform": "linux/amd64", + "files_scanned": 8854, + "files_read": 8854, + "findings": [ + { + "detector": "php2shell", + "kind": "php-runtime-web", + "path": "/usr/local/bin/php-cgi", + "verdict": "vulnerable", + "reason": "PHP 8.3.0 is below the fixed release 8.3.36; no package manager installed this file, so no vendor backport applies", + "message": "PHP 8.3.0 (cgi-fcgi)", + "method": "elf-strings", + "facts": { + "php_version": "8.3.0", + "sapi": "cgi-fcgi" + }, + "cves": [ + { + "id": "CVE-2026-109618", + "verdict": "vulnerable", + "reason": "PHP 8.3.0 is below the fixed release 8.3.36" + }, + { + "id": "CVE-2026-109619", + "verdict": "vulnerable", + "reason": "PHP 8.3.0 is below the fixed release 8.3.36" + } + ], + "fix": "Rebuild on PHP 8.3.36 or later (for the official Docker image, a php:8.3.36 tag), then redeploy.", + "severity": "critical", + "details": { + "path": "/usr/local/bin/php-cgi", + "php_version": "8.3.0", + "sapi": "cgi-fcgi", + "web_facing": true, + "status": "vulnerable", + "severity": "critical", + "reason": "PHP 8.3.0 is below the fixed release 8.3.36; no package manager installed this file, so no vendor backport applies", + "cves": [ + { + "id": "CVE-2026-109618", + "status": "vulnerable", + "reason": "PHP 8.3.0 is below the fixed release 8.3.36" + }, + { + "id": "CVE-2026-109619", + "status": "vulnerable", + "reason": "PHP 8.3.0 is below the fixed release 8.3.36" + } + ], + "os": { + "id": "debian", + "version_id": "12", + "codename": "bookworm", + "pretty_name": "Debian GNU/Linux 12 (bookworm)" + }, + "fix": "Rebuild on PHP 8.3.36 or later (for the official Docker image, a php:8.3.36 tag), then redeploy." + } + } + ], + "skipped": [ + { + "path": "/usr/src/php.tar.xz!/php-8.3.0/ext/bz2/tests/004_2.txt.bz2", + "reason": "bzip2: bzip2 data invalid: block checksum mismatch" + }, + { + "path": "/usr/src/php.tar.xz!/php-8.3.0/ext/bz2/tests/72613.bz2", + "reason": "bzip2: bzip2 data invalid: block checksum mismatch" + } + ], + "annotations": { + "os": { + "id": "debian", + "version_id": "12", + "codename": "bookworm", + "pretty_name": "Debian GNU/Linux 12 (bookworm)" + } + } + } + ] +} diff --git a/unittests/scans/earlylight/repo.json b/unittests/scans/earlylight/repo.json new file mode 100644 index 00000000000..1193fbd9e62 --- /dev/null +++ b/unittests/scans/earlylight/repo.json @@ -0,0 +1,112 @@ +{ + "tool": "earlylight", + "version": "v0.3.2-0.20261103043243-19c2e2404ae6", + "detectors": [ + { + "id": "brotli", + "title": "Brotli decoder vulnerable to CVE-2026-111497", + "cves": [ + "CVE-2026-111497" + ], + "advisories": [ + "https://github.com/advisories/GHSA-9v8h-qqgm-4m35", + "https://github.com/google/brotli/releases/tag/v1.2.1", + "https://nodejs.org/en/blog/vulnerability/october-2026-security-releases", + "https://github.com/dotnet/announcements/issues/494" + ], + "data": "Brotli decoder error-code tables (google/brotli 1.0.0 to 1.2.1) and the fixed Node.js, .NET and Bun releases" + }, + { + "id": "php2shell", + "title": "PHP2Shell: PHP vulnerable to CVE-2026-109618 and CVE-2026-109619", + "cves": [ + "CVE-2026-109618", + "CVE-2026-109619" + ], + "advisories": [ + "https://github.com/php/php-src/security/advisories/GHSA-w8hw-w9fx-j79w", + "https://github.com/php/php-src/security/advisories/GHSA-6wmf-4rv5-p3rp" + ], + "data": "distribution fix data from 2026-10-21T16:25:30Z" + } + ], + "targets": [ + { + "target": "repo", + "type": "repository", + "files_scanned": 6, + "files_read": 0, + "findings": [ + { + "detector": "brotli", + "kind": "node-pin", + "path": "app/Dockerfile", + "line": 1, + "verdict": "vulnerable", + "message": "Dockerfile FROM image node:20.0.0-slim pins Node.js 20.0.0, whose bundled Brotli decoder lacks the CVE-2026-111497 fix, on a release line with no fixed release (fixed in 22.23.4, 24.22.1 or 26.12.1).", + "method": "source", + "facts": { + "column": "11", + "current": "20.0.0", + "fixed": "22.23.4, 24.22.1 or 26.12.1", + "image": "node:20.0.0-slim", + "source": "node-image", + "upgrade": "major", + "upgrade_to": "22.23.4" + }, + "cves": [ + { + "id": "CVE-2026-111497", + "verdict": "vulnerable" + } + ], + "fix": "Moves Node.js 20.0.0 to 22.23.4, the nearest release line with the fix. This is a major upgrade, which can change behaviour: review it and run your tests before you merge, then rebuild and redeploy. The new image is used only once it is pulled and scanned clean on every platform the current one has." + }, + { + "detector": "php2shell", + "kind": "php-image", + "path": "web/Dockerfile", + "line": 1, + "verdict": "vulnerable", + "reason": "php:8.3.0-fpm carries a PHP release without the fix: linux/amd64 (PHP 8.3.0), linux/arm/v5 (PHP 8.3.0), linux/arm/v7 (PHP 8.3.0), linux/arm64/v8 (PHP 8.3.0), linux/386 (PHP 8.3.0), linux/mips64le (PHP 8.3.0), linux/ppc64le (PHP 8.3.0), linux/s390x (PHP 8.3.0)", + "message": "image php:8.3.0-fpm", + "method": "registry", + "facts": { + "image": "php:8.3.0-fpm", + "platforms": "linux/amd64 (PHP 8.3.0),linux/arm/v5 (PHP 8.3.0),linux/arm/v7 (PHP 8.3.0),linux/arm64/v8 (PHP 8.3.0),linux/386 (PHP 8.3.0),linux/mips64le (PHP 8.3.0),linux/ppc64le (PHP 8.3.0),linux/s390x (PHP 8.3.0)", + "reference": "dockerfile" + }, + "fix": "No automatic change: cannot tell which OS release docker.io/library/php:8.3.36-fpm or the original is built on for linux/amd64, linux/arm/v5, linux/arm/v7, linux/arm64/v8, linux/386, linux/ppc64le, linux/s390x (no org.opencontainers.image.base.name annotation or recognisable base layer), and the tag does not name one; pin a tag that names the OS release (for the php image: -bookworm, -trixie, -alpine3.21 ...)", + "severity": "critical", + "details": { + "file": "web/Dockerfile", + "line": 1, + "kind": "dockerfile", + "image": "php:8.3.0-fpm", + "status": "vulnerable", + "severity": "critical", + "reason": "php:8.3.0-fpm carries a PHP release without the fix: linux/amd64 (PHP 8.3.0), linux/arm/v5 (PHP 8.3.0), linux/arm/v7 (PHP 8.3.0), linux/arm64/v8 (PHP 8.3.0), linux/386 (PHP 8.3.0), linux/mips64le (PHP 8.3.0), linux/ppc64le (PHP 8.3.0), linux/s390x (PHP 8.3.0)", + "platforms": [ + "linux/amd64 (PHP 8.3.0)", + "linux/arm/v5 (PHP 8.3.0)", + "linux/arm/v7 (PHP 8.3.0)", + "linux/arm64/v8 (PHP 8.3.0)", + "linux/386 (PHP 8.3.0)", + "linux/mips64le (PHP 8.3.0)", + "linux/ppc64le (PHP 8.3.0)", + "linux/s390x (PHP 8.3.0)" + ], + "fix": "No automatic change: cannot tell which OS release docker.io/library/php:8.3.36-fpm or the original is built on for linux/amd64, linux/arm/v5, linux/arm/v7, linux/arm64/v8, linux/386, linux/ppc64le, linux/s390x (no org.opencontainers.image.base.name annotation or recognisable base layer), and the tag does not name one; pin a tag that names the OS release (for the php image: -bookworm, -trixie, -alpine3.21 ...)" + } + } + ], + "skipped": [ + { + "path": "compose.yaml:3 registry.example.internal/team/php-api:8.3", + "reason": "image not checked: registry.example.internal/team/php-api:8.3: network: Get \"https://registry.example.internal/v2/\": dial tcp: lookup registry.example.internal: no such host" + } + ], + "incomplete": "1 image reference(s) could not be checked because a registry failed: compose.yaml:3 registry.example.internal/team/php-api:8.3 (network)" + } + ] +} diff --git a/unittests/tools/test_earlylight_parser.py b/unittests/tools/test_earlylight_parser.py new file mode 100644 index 00000000000..4268c757022 --- /dev/null +++ b/unittests/tools/test_earlylight_parser.py @@ -0,0 +1,257 @@ +import io +import json + +from dojo.models import Finding, Test +from dojo.tools.earlylight.parser import EarlylightParser +from unittests.dojo_test_case import DojoTestCase, get_unit_tests_scans_path + +# The sample reports in unittests/scans/earlylight are real `earlylight scan ... --format json` +# output from Early Light main at 19c2e24 (after v0.3.1), trimmed to a few findings each with every +# "skipped" list cut to two entries: +# no_findings.json scan fs on a directory with one text file (path renamed to /srv/app) +# one_vuln.json scan image php:8.3.0-cli, one finding kept +# many_vulns.json scan image php:8.3.0-cli, five of its nine findings kept +# needs_review.json scan image --detector php2shell trafex/php-nginx:2.0.1 +# cleared_only.json scan image --detector php2shell php:8.3.36-cli +# many_targets.json scan image --detector php2shell,brotli php:8.3.36-cli trafex/php-nginx:2.0.1 node:20.0.0-slim +# repo.json scan repo on a source tree with two Dockerfiles and a Compose file + + +class TestEarlylightParser(DojoTestCase): + + def parse(self, filename): + with (get_unit_tests_scans_path("earlylight") / filename).open(encoding="utf-8") as testfile: + return EarlylightParser().get_findings(testfile, Test()) + + def test_scan_type(self): + parser = EarlylightParser() + self.assertEqual(["Early Light Scan"], parser.get_scan_types()) + self.assertEqual("Early Light Scan", parser.get_label_for_scan_types("Early Light Scan")) + self.assertIn("--format json", parser.get_description_for_scan_types("Early Light Scan")) + + def test_no_findings(self): + self.assertEqual(0, len(self.parse("no_findings.json"))) + + def test_cleared_findings_are_skipped(self): + # four fixed php2shell findings and nothing else + self.assertEqual(0, len(self.parse("cleared_only.json"))) + + def test_one_vuln(self): + findings = self.parse("one_vuln.json") + self.assertEqual(1, len(findings)) + finding = findings[0] + self.assertEqual("php2shell/php-runtime-web: PHP 8.3.0 (cgi-fcgi)", finding.title) + self.assertEqual("Critical", finding.severity) + self.assertEqual("php2shell/php-runtime-web", finding.vuln_id_from_tool) + self.assertEqual(["CVE-2026-109618", "CVE-2026-109619"], finding.unsaved_vulnerability_ids) + self.assertEqual(["earlylight"], finding.unsaved_tags) + self.assertEqual("/usr/local/bin/php-cgi", finding.file_path) + self.assertIsNone(finding.line) + self.assertEqual(416, finding.cwe) + self.assertEqual("php", finding.component_name) + self.assertEqual("8.3.0", finding.component_version) + self.assertTrue(finding.static_finding) + self.assertFalse(finding.dynamic_finding) + self.assertTrue(finding.fix_available) + self.assertEqual( + "Rebuild on PHP 8.3.36 or later (for the official Docker image, a php:8.3.36 tag), then redeploy.", + finding.mitigation, + ) + self.assertEqual( + "https://github.com/php/php-src/security/advisories/GHSA-w8hw-w9fx-j79w\n" + "https://github.com/php/php-src/security/advisories/GHSA-6wmf-4rv5-p3rp", + finding.references, + ) + self.assertEqual("fc2cb38cd56257fb9c612aaff9a054e2c21aabd180dd317bdd618e9ec96b240e", finding.unique_id_from_tool) + self.assertIn("**Verdict:** vulnerable", finding.description) + self.assertIn("**Target:** php:8.3.0-cli (image, sha256:33b1f2a2edaa92c73b41cbb078e8c1abce9b4ebe41af870415c6361830667da7, linux/amd64)", finding.description) + self.assertIn("**Path:** /usr/local/bin/php-cgi", finding.description) + self.assertIn("**Method:** elf-strings", finding.description) + self.assertIn("- sapi: cgi-fcgi", finding.description) + self.assertIn("- CVE-2026-109618: vulnerable (PHP 8.3.0 is below the fixed release 8.3.36)", finding.description) + self.assertNotIn("needs-review", finding.unsaved_tags) + + def test_many_vulns(self): + findings = self.parse("many_vulns.json") + self.assertEqual(5, len(findings)) + for finding in findings: + self.assertIn(finding.severity, Finding.SEVERITIES) + self.assertEqual(["earlylight"], finding.unsaved_tags) + self.assertEqual(64, len(finding.unique_id_from_tool)) + self.assertIn("**Target:** php:8.3.0-cli", finding.description) + self.assertEqual(5, len({f.unique_id_from_tool for f in findings})) + + with self.subTest(kind="brotli/decoder-library"): + finding = findings[0] + self.assertEqual("brotli/decoder-library", finding.vuln_id_from_tool) + self.assertEqual("Critical", finding.severity) # from the kind's 9.2 score + self.assertEqual(["CVE-2026-111497"], finding.unsaved_vulnerability_ids) + self.assertEqual("libbrotlidec", finding.component_name) + self.assertIsNone(finding.component_version) + self.assertEqual(252, finding.cwe) + self.assertEqual("/usr/lib/x86_64-linux-gnu/libbrotlidec.so.1.0.9", finding.file_path) + self.assertIn("https://github.com/advisories/GHSA-9v8h-qqgm-4m35", finding.references) + + with self.subTest(kind="emudhra/path-root"): + # two CAs in one bundle: same kind and path, told apart by the certificate + first, second = findings[1], findings[2] + for finding in (first, second): + self.assertEqual("emudhra/path-root", finding.vuln_id_from_tool) + self.assertEqual("High", finding.severity) # level error, no score + self.assertEqual("/etc/ssl/certs/ca-certificates.crt", finding.file_path) + # the detector names CVE-2026-111943, but the trust finding itself has none + self.assertIsNone(finding.unsaved_vulnerability_ids) + self.assertEqual(0, finding.cwe) # Finding default: no CWE + self.assertIsNone(finding.mitigation) + self.assertFalse(finding.fix_available) + self.assertEqual("https://bugzilla.mozilla.org/show_bug.cgi?id=2083449", finding.references) + self.assertNotEqual(first.unique_id_from_tool, second.unique_id_from_tool) + self.assertIn("AAA Certificate Services", first.title) + self.assertIn("- spki_sha256: BD153ED7B0434F6886B17BCE8BBE84ED340C7132D702A8F4FA318F756ECBD6F3", first.description) + + with self.subTest(kind="php2shell/php-runtime-cli"): + finding = findings[3] + self.assertEqual("php2shell/php-runtime-cli", finding.vuln_id_from_tool) + self.assertEqual("High", finding.severity) # the finding's own "high" + self.assertEqual(["CVE-2026-109618", "CVE-2026-109619"], finding.unsaved_vulnerability_ids) + self.assertEqual("/usr/local/bin/php", finding.file_path) + + def test_unique_id_is_stable(self): + first = [f.unique_id_from_tool for f in self.parse("many_vulns.json")] + second = [f.unique_id_from_tool for f in self.parse("many_vulns.json")] + self.assertEqual(first, second) + # the same PHP binary found in a report of a different size keeps its id + one = self.parse("one_vuln.json")[0] + self.assertIn(one.unique_id_from_tool, first) + + def test_needs_review(self): + findings = self.parse("needs_review.json") + self.assertEqual(2, len(findings)) + for finding in findings: + self.assertEqual(["earlylight", "needs-review"], finding.unsaved_tags) + self.assertEqual(["CVE-2026-109618", "CVE-2026-109619"], finding.unsaved_vulnerability_ids) + self.assertIn("**Needs review:** Early Light could not settle this finding.", finding.description) + self.assertIn( + "this tool has no fix data for that package stream, and the vendor may have backported the fix", + finding.description, + ) + self.assertIn("- CVE-2026-109618: unknown (installed by php8", finding.description) + self.assertIn("**Target:** trafex/php-nginx:2.0.1 (image,", finding.description) + self.assertNotIn("**Verdict:** vulnerable", finding.description) + cli, web = findings + # severity "review" is not a severity: the kind's table entry decides + self.assertEqual("High", cli.severity) + self.assertEqual("Critical", web.severity) + self.assertEqual("php2shell/php-runtime-web", web.vuln_id_from_tool) + # the installing OS package is the component + self.assertEqual("php8-fpm", web.component_name) + self.assertEqual("8.0.2-r0", web.component_version) + self.assertEqual( + "Check the security advisories for php8 on Alpine Linux v3.13 and upgrade the package if a fix is listed.", + web.mitigation, + ) + + def test_many_targets(self): + findings = self.parse("many_targets.json") + # php:8.3.36-cli: five fixed, skipped. trafex/php-nginx:2.0.1: two unknown, one vulnerable. + # node:20.0.0-slim: one vulnerable. + self.assertEqual(4, len(findings)) + self.assertEqual( + ["php2shell/php-runtime-cli", "php2shell/php-runtime-web", "brotli/decoder-library", "brotli/bundled-decoder"], + [f.vuln_id_from_tool for f in findings], + ) + self.assertEqual( + [["earlylight", "needs-review"], ["earlylight", "needs-review"], ["earlylight"], ["earlylight"]], + [f.unsaved_tags for f in findings], + ) + node = findings[3] + self.assertIn("**Target:** node:20.0.0-slim (image,", node.description) + self.assertEqual("Critical", node.severity) + self.assertEqual("node", node.component_name) + self.assertEqual("20.0.0", node.component_version) + self.assertEqual("/usr/local/bin/node", node.file_path) + self.assertEqual(["CVE-2026-111497"], node.unsaved_vulnerability_ids) + self.assertEqual("Node.js 20 has no fixed release: move to 22.23.4, 24.22.1 or 26.12.1.", node.mitigation) + brotli = findings[2] + self.assertIn("**Target:** trafex/php-nginx:2.0.1", brotli.description) + self.assertEqual("/usr/lib/libbrotlidec.so.1.0.9", brotli.file_path) + + def test_repo(self): + findings = self.parse("repo.json") + self.assertEqual(2, len(findings)) + node_pin, php_image = findings + + self.assertEqual("brotli/node-pin", node_pin.vuln_id_from_tool) + self.assertEqual("app/Dockerfile", node_pin.file_path) + self.assertEqual(1, node_pin.line) + self.assertEqual("Critical", node_pin.severity) + self.assertEqual(["CVE-2026-111497"], node_pin.unsaved_vulnerability_ids) + self.assertIn("**Target:** repo (repository)", node_pin.description) + self.assertIn("**Path:** app/Dockerfile:1", node_pin.description) + self.assertIsNone(node_pin.component_name) + + self.assertEqual("php2shell/php-image", php_image.vuln_id_from_tool) + self.assertEqual("web/Dockerfile", php_image.file_path) + self.assertEqual(1, php_image.line) + self.assertEqual("Critical", php_image.severity) + self.assertEqual("php2shell/php-image: image php:8.3.0-fpm", php_image.title) + # php-image findings carry no CVE list in the report + self.assertIsNone(php_image.unsaved_vulnerability_ids) + + def test_severity_rules(self): + report = { + "tool": "earlylight", + "version": "test", + "detectors": [], + "targets": [{ + "target": "example.com:443", + "type": "host", + "findings": [ + {"detector": "emudhra", "kind": "unpatched-go", "path": "example.com:443", "verdict": "vulnerable", "message": "m"}, + {"detector": "brotli", "kind": "go-port", "path": "/app", "verdict": "vulnerable", "message": "m", "severity": "low"}, + {"detector": "future", "kind": "thing", "path": "/a", "verdict": "vulnerable", "message": "m"}, + {"detector": "future", "kind": "thing", "path": "/b", "verdict": "unknown", "message": "m", "severity": "review"}, + {"detector": "future", "kind": "thing", "path": "/c", "verdict": "managed", "message": "m"}, + {"detector": "future", "kind": "thing", "path": "/d", "verdict": "inactive", "message": "m"}, + ], + }], + } + findings = EarlylightParser().get_findings(io.StringIO(json.dumps(report)), Test()) + self.assertEqual(["Medium", "Low", "High", "Medium"], [f.severity for f in findings]) + self.assertTrue(findings[0].dynamic_finding) + self.assertFalse(findings[0].static_finding) + self.assertIsNone(findings[0].references) + + def test_reported_kinds_win_over_the_table(self): + report = { + "detectors": [{"id": "brotli", "kinds": [ + {"id": "go-port", "title": "t", "level": "warning", "severity": "7.1"}, + {"id": "not-checked", "title": "t", "level": "note"}, + ]}, {"id": "future", "kinds": [{"id": "thing", "title": "t", "level": "error", "severity": "3.0"}]}], + "targets": [{"target": "/srv", "type": "filesystem", "findings": [ + {"detector": "brotli", "kind": "go-port", "path": "/a", "verdict": "vulnerable", "message": "m"}, + {"detector": "brotli", "kind": "not-checked", "path": "/b", "verdict": "vulnerable", "message": "m"}, + {"detector": "future", "kind": "thing", "path": "/c", "verdict": "vulnerable", "message": "m"}, + {"detector": "future", "kind": "thing", "path": "/d", "verdict": "vulnerable", "message": "m", "severity": "critical"}, + ]}], + } + findings = EarlylightParser().get_findings(io.StringIO(json.dumps(report)), Test()) + self.assertEqual(["High", "Info", "Low", "Critical"], [f.severity for f in findings]) + self.assertEqual(252, findings[0].cwe) + + def test_identical_findings_get_distinct_ids(self): + item = {"detector": "brotli", "kind": "dependency", "path": "requirements.txt", "verdict": "vulnerable", "message": "m"} + report = {"detectors": [], "targets": [{"target": "src", "type": "repository", "findings": [item, dict(item, line=9)]}]} + findings = EarlylightParser().get_findings(io.StringIO(json.dumps(report)), Test()) + self.assertEqual(2, len({f.unique_id_from_tool for f in findings})) + + def test_bytes_input(self): + with (get_unit_tests_scans_path("earlylight") / "one_vuln.json").open("rb") as testfile: + findings = EarlylightParser().get_findings(testfile, Test()) + self.assertEqual(1, len(findings)) + + def test_invalid_input(self): + with self.assertRaises(ValueError): + EarlylightParser().get_findings(io.StringIO("not json"), Test()) + with self.assertRaises(ValueError): + EarlylightParser().get_findings(io.StringIO('{"runs": []}'), Test()) From bf6cd53f6b276bbed4128ab9411481466da796d0 Mon Sep 17 00:00:00 2001 From: Cody Maffucci <46459665+Maffooch@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:35:51 +0000 Subject: [PATCH 07/61] docs(changelog): add DefectDojo Pro 3.5.102 release notes (#16820) Co-authored-by: Claude Opus 5.5 --- docs/content/releases/pro/changelog.md | 31 ++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/docs/content/releases/pro/changelog.md b/docs/content/releases/pro/changelog.md index b5731857da9..aee16027b16 100644 --- a/docs/content/releases/pro/changelog.md +++ b/docs/content/releases/pro/changelog.md @@ -22,6 +22,37 @@ For features that are deprecated, scheduled for removal or already removed, see ## November 2026: v3.5 +### November 12, 2026: v3.5.102 + +New features: +* **(Triage Engine)** The **On Rule Complete** trigger runs a rule each time another rule finishes a successful run, so rules can run in a fixed order whatever they change. The rule sweeps its own scope, as a Manual Run does, and a failed run starts nothing. See [On Rule Complete](/automation/triage_engine/node_reference/#on-rule-complete) and [Running rules in order](/automation/triage_engine/runs/#running-rules-in-order). + +Enhancements: +* **(Triage Engine)** With **Restrict Triage Engine Emails to DefectDojo Users** on, a **Send an Email** batch also reaches recipients who can see only part of it. Each recipient gets an email that lists just the Findings, Assets or alerts they can see, with a count that matches. See [Restricting recipients to DefectDojo users](/automation/triage_engine/node_reference/#restricting-recipients-to-defectdojo-users). +* **(AI)** AI Guardrails keeps Prompt Activity for a set number of days, 90 by default, which a superuser changes on the new **Settings** tab. A **Daily Activity** chart keeps each day's counts after the events themselves are deleted. See [Prompt Activity](/triage_findings/findings_workflows/pro__ai_guardrails/#prompt-activity). +* **(Connectors)** When the license is refusing connector syncs, the Connections list and a connection's records page say so and link to the License page. See [Managing Operations](/connectors/upstream/manage_operations/). +* **(Performance)** History tabs load faster on instances with a long audit history. Adding a tag to many Findings at once, the Tests API with Jira-linked Finding Groups or `response_fields`, Findings API pages that include duplicates, the Triage Engine **Rules** list, and the close step of a large connector sync are faster. + +Bug fixes: +* **(Connectors)** A large connector sync that is cut off by its time limit, a deploy or a lost worker continues where it stopped. It finishes, closes the Findings the tool no longer reports and records the sync in Import History. See [Sync Operations](/connectors/upstream/manage_operations/#sync-operations). +* **(Triage Engine)** Turning a rule on or off takes effect right away. A rule that changes Findings and then deletes them in the same run completes. A rule that sets an Organization or parent Asset by a computed name skips a name an archived Organization or Asset holds instead of failing the run. The **Import Rules** preview runs again when **Owned By** changes while a preview is running. See [Runs](/automation/triage_engine/runs/) and [Importing rules](/automation/triage_engine/sharing_and_versions/#importing-rules). +* **(Risk Acceptance)** Closing a risk-accepted Finding marks its Endpoint statuses mitigated. See [Risk Acceptances](/triage_findings/findings_workflows/pro__risk_acceptance/). +* **(Compliance)** Evidence packs for an assessment that covers several Assets or a whole Organization can be generated, opened and downloaded. The CMMC Assessment page opens an Asset's newest assessment, the same one the Asset's Compliance summary reports. See [Federal Compliance](/federal_compliance/) and [CMMC Level 2 Assessments](/federal_compliance/cmmc_assessments/). +* **(Jira)** A classic Jira instance whose configuration name is longer than 255 characters migrates to a Jira Downstream Connector. See [Migrating to the Jira Downstream Connector](/connectors/downstream/pro__jira_guide/#migrating-to-the-jira-downstream-connector). +* **(Dashboards)** A Graph over a date labels each period by its size, in the reader's language, instead of with an ISO date. A short Graph tile keeps room for its chart. The **Regulatory Scope** widget answers a malformed regulations value with a validation error instead of a server error. See [Graph options](/metrics_reports/dashboards/custom-dashboards/#graph-options). +* **(Findings)** A Component with a blank name opens its own Findings from **All Components** and from the Asset's **Components** widget. See [Components](/asset_modelling/components/pro__components/). +* **(UI)** The severity donut's ring labels stay off its legend. The source toggle on a Markdown field no longer covers the text. A failed table preference read or Location map save shows one toast instead of several. The column filter menus on the Components and Support Requests tables show their match modes in the app language, and Dedupe Pool counts use the app language's number format. + +Security fixes: +* **(Hardening)** Date filters refuse a **Past Days** value outside the supported range. `dojo-deploy` is built with a Go release that includes the latest security fixes. See [Filter Index](/navigation/pro__filter_index/#how-date-filters-are-evaluated) and [Keeping the CLI up to date](/get_started/pro/onprem/dojo_deploy/#keeping-the-cli-up-to-date). + +Behavior changes: +* **(Authentication)** Nobody is asked to accept the End User License Agreement after signing in. The organization's subscription agreement already covers its users. + +Upgrade notes: +* **(Deployment)** The upgrade adds indexes to the audit history tables, which can take a minute or more on an instance with millions of Findings. One read of an object's History tab is limited by `DD_OBJECT_HISTORY_STATEMENT_TIMEOUT`, 30 seconds by default, and `0` leaves it only the web statement limit. See [Hardware Sizing](/get_started/pro/onprem/hardware_sizing/#sizing-and-tuning-the-application-tier). +* **(Helm)** With `networkPolicy.profile: aggressive`, the Celery imports worker, the OSCAL validator, the Orchestrator workers and the `helm test` pods get the network access they need, including to the embedded Redis. See [Network Policies](/get_started/pro/onprem/kubernetes/installing_on_kubernetes/#network-policies) and [DefectDojo Pro Upgrade Guide (Kubernetes / Helm)](/get_started/pro/onprem/kubernetes/upgrading_on_kubernetes/). + ### November 11, 2026: v3.5.101 **NOTE: Endpoints are deprecated and will be removed in 3.7.0 on January 4, 2027.** From 3188b24c1191d3f2c8da9ba6c0933ad70825151e Mon Sep 17 00:00:00 2001 From: Cody Maffucci <46459665+Maffooch@users.noreply.github.com> Date: Thu, 12 Nov 2026 18:54:43 +0000 Subject: [PATCH 08/61] perf(tags): index UPPER(name) on every tag table (#16801) Tag fields are case-insensitive, so tagulous finds each tag it adds with get_or_create(name__iexact=...), which compiles to UPPER("name"::text) = UPPER(%s). The unique index on name cannot serve that, so every tag added to an object read the whole tag table. Tag rows pile up on an instance whose pipelines create, tag and delete findings, because a tag row is not deleted when its last finding is. On a 3,000,000-row finding tag table one lookup took 1.6 s, and a finding PATCH that set 12 tags spent 19 s in SQL. dojo.0308 adds a text_pattern_ops btree on UPPER(name) to each of the 14 tag tables, so the lookup is an index scan (0.03 ms) and the istartswith tag search can use it too. The migration builds CONCURRENTLY outside a transaction (plainly inside one), drops an INVALID leftover of an interrupted build first, and ANALYZEs each table. It changes no migration state: the tag models are generated by tagulous and declare no indexes. Co-authored-by: Claude Opus 5.5 --- .../0308_tag_name_upper_index.py | 87 ++++++++++ unittests/test_tag_name_upper_index.py | 163 ++++++++++++++++++ 2 files changed, 250 insertions(+) create mode 100644 dojo/db_migrations/0308_tag_name_upper_index.py create mode 100644 unittests/test_tag_name_upper_index.py diff --git a/dojo/db_migrations/0308_tag_name_upper_index.py b/dojo/db_migrations/0308_tag_name_upper_index.py new file mode 100644 index 00000000000..4c6bc073b02 --- /dev/null +++ b/dojo/db_migrations/0308_tag_name_upper_index.py @@ -0,0 +1,87 @@ +from django.db import migrations + +# Every tagulous tag table. Their models are generated by tagulous at runtime, so the index is +# created with SQL only: declaring it in the migration state would make makemigrations emit a +# RemoveIndex for each table, since the generated models declare no indexes. +TAG_TABLES = ( + "dojo_tagulous_app_analysis_tags", + "dojo_tagulous_endpoint_inherited_tags", + "dojo_tagulous_endpoint_tags", + "dojo_tagulous_engagement_inherited_tags", + "dojo_tagulous_engagement_tags", + "dojo_tagulous_finding_inherited_tags", + "dojo_tagulous_finding_tags", + "dojo_tagulous_finding_template_tags", + "dojo_tagulous_location_inherited_tags", + "dojo_tagulous_location_tags", + "dojo_tagulous_objects_product_tags", + "dojo_tagulous_product_tags", + "dojo_tagulous_test_inherited_tags", + "dojo_tagulous_test_tags", +) + + +def index_name(table): + return "idx_" + table.removeprefix("dojo_") + "_upper_name" + + +def add_indexes(apps, schema_editor): + connection = schema_editor.connection + if connection.vendor != "postgresql": + return + # CONCURRENTLY cannot run in a transaction block; a caller's transaction (a test, a tenant + # migrated under an outer atomic block) gets a plain build. + concurrently = "" if connection.in_atomic_block else "CONCURRENTLY " + with connection.cursor() as cursor: + for table in TAG_TABLES: + name = index_name(table) + # An interrupted CONCURRENTLY build leaves an INVALID index that IF NOT EXISTS would keep. + cursor.execute( + "SELECT indisvalid FROM pg_index WHERE indexrelid = to_regclass(%s)", + [name], + ) + row = cursor.fetchone() + if row is not None and not row[0]: + cursor.execute(f'DROP INDEX {concurrently}IF EXISTS "{name}"') + cursor.execute( + f'CREATE INDEX {concurrently}IF NOT EXISTS "{name}" ' + f'ON "{table}" (UPPER("name"::text) text_pattern_ops)', + ) + # Statistics for the new expression, so the planner can estimate a lookup on it. + cursor.execute(f'ANALYZE "{table}"') + + +def remove_indexes(apps, schema_editor): + connection = schema_editor.connection + if connection.vendor != "postgresql": + return + concurrently = "" if connection.in_atomic_block else "CONCURRENTLY " + with connection.cursor() as cursor: + for table in TAG_TABLES: + cursor.execute(f'DROP INDEX {concurrently}IF EXISTS "{index_name(table)}"') + + +class Migration(migrations.Migration): + + """ + Index UPPER(name) on every tag table. + + Tag fields are case-insensitive, so tagulous finds each tag it adds with + get_or_create(name__iexact=...), which compiles to UPPER("name"::text) = UPPER(%s). The unique + index on name cannot serve that, so every tag added to an object read the whole tag table. On an + instance whose pipelines create, tag and delete findings all the time, tag rows pile up (a tag is + not deleted when its last finding is), and a finding PATCH that set 12 tags spent 18 s reading a + 3,000,000-row tag table twelve times. The importer, the bulk tag helpers and tag search + (istartswith, which text_pattern_ops also serves) use the same expression. Non-atomic so each + index can build CONCURRENTLY without blocking tag writes; safe to re-run. + """ + + atomic = False + + dependencies = [ + ("dojo", "0307_event_history_lookup_indexes"), + ] + + operations = [ + migrations.RunPython(add_indexes, remove_indexes), + ] diff --git a/unittests/test_tag_name_upper_index.py b/unittests/test_tag_name_upper_index.py new file mode 100644 index 00000000000..9d88e5abdcb --- /dev/null +++ b/unittests/test_tag_name_upper_index.py @@ -0,0 +1,163 @@ +""" +Case-insensitive tag lookups are served by an index on UPPER(name), on every tag table. + +Tag fields are case-insensitive, so tagulous finds each tag it adds with ``get_or_create(name__iexact=...)``, +which compiles to ``UPPER("name"::text) = UPPER(%s)``; the importer, the bulk tag helpers and tag search +(``istartswith``) use the same expression. The unique index on ``name`` cannot serve it, so on an instance whose +tag table had grown to millions of rows every tag added to a finding was a sequential scan of that table +(a PATCH setting 12 tags: 18 s of SQL). ``dojo.0308`` indexes ``UPPER(name)`` on every tag table. +""" + +import importlib +from unittest import mock + +from django.apps import apps +from django.db import connection +from django_tagulous.models import BaseTagModel + +from dojo.models import Finding, Product, Product_Type +from unittests.dojo_test_case import DojoTestCase + +migration = importlib.import_module("dojo.db_migrations.0308_tag_name_upper_index") + + +def _tag_tables(): + return { + model._meta.db_table for model in apps.get_models(include_auto_created=True) if issubclass(model, BaseTagModel) + } + + +def _valid_upper_name_indexes(): + """{table: index name} for every valid index on UPPER(name) of a tag table.""" + with connection.cursor() as cursor: + cursor.execute( + """ + SELECT c.relname, ic.relname + FROM pg_index i + JOIN pg_class c ON c.oid = i.indrelid + JOIN pg_class ic ON ic.oid = i.indexrelid + WHERE i.indisvalid AND c.relname LIKE 'dojo_tagulous_%' + AND pg_get_indexdef(i.indexrelid) LIKE '%(upper((name)::text) text_pattern_ops)' + """, + ) + return dict(cursor.fetchall()) + + +def _plan(queryset): + """ + The plan of ``queryset`` on a finding tag table of realistic size. + + On the few rows of a test database the planner may as well read the whole unique index on + ``name`` and filter, so the table first gets 5,000 tags and fresh statistics (both rolled back + with the test's transaction). + """ + tag_model = Finding.tags.tag_model + tag_model.objects.bulk_create( + tag_model(name=f"upper-index-filler-{n}", slug=f"upper-index-filler-{n}", count=0, protected=False) + for n in range(5000) + ) + sql, params = queryset.query.sql_with_params() + with connection.cursor() as cursor: + cursor.execute(f"ANALYZE {tag_model._meta.db_table}") + cursor.execute(f"EXPLAIN {sql}", params) + return "\n".join(row[0] for row in cursor.fetchall()) + + +class TagNameUpperIndexTest(DojoTestCase): + def test_every_tag_table_has_a_valid_upper_name_index(self): + tables = _tag_tables() + self.assertIn("dojo_tagulous_finding_tags", tables) + self.assertEqual(set(migration.TAG_TABLES), tables, "a new tag field needs its table in dojo.0308's list") + indexes = _valid_upper_name_indexes() + self.assertEqual(set(), tables - set(indexes)) + self.assertEqual({table: migration.index_name(table) for table in tables}, indexes) + + def test_the_lookup_that_adds_a_tag_uses_the_index(self): + self.assertFalse(Finding.tags.tag_options.case_sensitive) + plan = _plan(Finding.tags.tag_model.objects.filter(name__iexact="Some-Tag")) + self.assertIn("using idx_tagulous_finding_tags_upper_name", plan) + self.assertIn("Index Cond: (upper((name)::text) = 'SOME-TAG'::text)", plan) + + def test_a_prefix_tag_search_uses_the_index(self): + plan = _plan(Finding.tags.tag_model.objects.filter(name__istartswith="some-")) + self.assertIn("using idx_tagulous_finding_tags_upper_name", plan) + self.assertIn("Index Cond: ((upper((name)::text) ~>=~ 'SOME-'::text)", plan) + + def test_adding_a_case_variant_reuses_the_existing_tag(self): + prod_type = Product_Type.objects.create(name="Upper Index Type") + first, second = ( + Product.objects.create(name=f"Upper Index Product {n}", description="tags", prod_type=prod_type) + for n in (1, 2) + ) + tag_model = Product.tags.tag_model + first.tags.set(["Upper-Index-Tag"]) + existing = tag_model.objects.get(name__iexact="upper-index-tag") + second.tags.set(["UPPER-INDEX-TAG"]) + self.assertEqual( + [existing.id], list(tag_model.objects.filter(name__iexact="upper-index-tag").values_list("id", flat=True)), + ) + self.assertEqual([existing.id], list(second.tags.values_list("id", flat=True))) + self.assertEqual(2, tag_model.objects.get(id=existing.id).count) + + +class TagNameUpperIndexMigrationTest(DojoTestCase): + + """The migration's build choices, driven with a stand-in connection (a real INVALID index only comes out of an interrupted concurrent build).""" + + @staticmethod + def _schema_editor(*, vendor="postgresql", in_atomic_block=False, indisvalid=None): + cursor = mock.MagicMock() + cursor.fetchone.return_value = None if indisvalid is None else (indisvalid,) + db = mock.MagicMock(vendor=vendor, in_atomic_block=in_atomic_block) + db.cursor.return_value.__enter__.return_value = cursor + return mock.MagicMock(connection=db), cursor + + @staticmethod + def _statements(cursor): + return [call.args[0] for call in cursor.execute.call_args_list] + + def test_it_builds_concurrently_outside_a_transaction_and_analyzes_each_table(self): + schema_editor, cursor = self._schema_editor() + migration.add_indexes(apps, schema_editor) + statements = self._statements(cursor) + self.assertIn( + 'CREATE INDEX CONCURRENTLY IF NOT EXISTS "idx_tagulous_finding_tags_upper_name" ' + 'ON "dojo_tagulous_finding_tags" (UPPER("name"::text) text_pattern_ops)', + statements, + ) + self.assertIn('ANALYZE "dojo_tagulous_finding_tags"', statements) + self.assertFalse(any(statement.startswith("DROP") for statement in statements)) + + def test_it_builds_plainly_inside_a_transaction(self): + schema_editor, cursor = self._schema_editor(in_atomic_block=True) + migration.add_indexes(apps, schema_editor) + creates = [statement for statement in self._statements(cursor) if statement.startswith("CREATE")] + self.assertEqual(len(migration.TAG_TABLES), len(creates)) + self.assertTrue(all(statement.startswith("CREATE INDEX IF NOT EXISTS") for statement in creates)) + + def test_it_drops_an_invalid_leftover_before_building_again(self): + schema_editor, cursor = self._schema_editor(indisvalid=False) + migration.add_indexes(apps, schema_editor) + self.assertIn( + 'DROP INDEX CONCURRENTLY IF EXISTS "idx_tagulous_finding_tags_upper_name"', + self._statements(cursor), + ) + + def test_it_keeps_a_valid_index(self): + schema_editor, cursor = self._schema_editor(indisvalid=True) + migration.add_indexes(apps, schema_editor) + self.assertFalse(any(statement.startswith("DROP") for statement in self._statements(cursor))) + + def test_it_drops_every_index_when_unapplied(self): + schema_editor, cursor = self._schema_editor() + migration.remove_indexes(apps, schema_editor) + self.assertEqual( + [f'DROP INDEX CONCURRENTLY IF EXISTS "{migration.index_name(table)}"' for table in migration.TAG_TABLES], + self._statements(cursor), + ) + + def test_it_does_nothing_on_another_database(self): + for operation in (migration.add_indexes, migration.remove_indexes): + schema_editor, cursor = self._schema_editor(vendor="sqlite") + operation(apps, schema_editor) + cursor.execute.assert_not_called() From 5dece890ad9bbce89a0cbb5d8fd4d4bd8e1ac025 Mon Sep 17 00:00:00 2001 From: Cody Maffucci <46459665+Maffooch@users.noreply.github.com> Date: Thu, 12 Nov 2026 19:59:18 +0000 Subject: [PATCH 09/61] perf(auditlog): give the history index builds the memory and workers to build fast (#16823) 0307 built each index on a large event table CONCURRENTLY under the database's default maintenance_work_mem. On an instance with about 2.9M findings that held an upgrade for about two hours, almost all of it three builds on the Finding event table. At the 64 MB default each build's sort spills to disk, and since Postgres gives a build one parallel worker per 32 MB of maintenance_work_mem, each build also ran with one worker. Each concurrent build now runs with maintenance_work_mem = 512MB and max_parallel_maintenance_workers = 4 in its own session, never lowering a session that already has more, and the session's settings are restored after the build. Indexes are built smallest table first, so a run that fails on a large table leaves every small one built. Migration state is unchanged. Co-authored-by: Claude Opus 5.5 --- .../0307_event_history_lookup_indexes.py | 52 +++++++++-- unittests/test_auditlog_history_indexes.py | 92 +++++++++++++++++++ 2 files changed, 136 insertions(+), 8 deletions(-) diff --git a/dojo/db_migrations/0307_event_history_lookup_indexes.py b/dojo/db_migrations/0307_event_history_lookup_indexes.py index 10752da832c..85ee330f968 100644 --- a/dojo/db_migrations/0307_event_history_lookup_indexes.py +++ b/dojo/db_migrations/0307_event_history_lookup_indexes.py @@ -35,6 +35,11 @@ class AddHistoryLookupIndexes(Operation): plain_build_lock_timeout = "2s" concurrent_build_lock_timeout = "1h" concurrent_build_wait_seconds = 3600 + # A concurrent build on a large table is a full heap scan plus a sort. Postgres gives a build one + # parallel worker per 32 MB of maintenance_work_mem at most, so the 64 MB default spills the sort + # and allows a single worker. Raised for the build's own session only, never lowered. + concurrent_build_maintenance_work_mem = "512MB" + concurrent_build_parallel_workers = 4 poll_seconds = 5 def __init__(self, indexes): @@ -64,14 +69,30 @@ def _concurrent_allowed(schema_editor): return connection.vendor == "postgresql" and not connection.in_atomic_block def database_forwards(self, app_label, schema_editor, from_state, to_state): + concurrent = self._concurrent_allowed(schema_editor) + todo = [] for model_name, index in self.indexes: model = to_state.apps.get_model(app_label, model_name) - if not self.allow_migrate_model(schema_editor.connection.alias, model): - continue - if self._concurrent_allowed(schema_editor): - self._add(schema_editor, model, index) - else: + if self.allow_migrate_model(schema_editor.connection.alias, model): + todo.append((model, index)) + if not concurrent: + for model, index in todo: schema_editor.add_index(model, index) + return + # Smallest tables first, so a run that fails or times out on a large table leaves every small + # one built, and a re-run only has the large ones left. sorted() keeps a table's own order. + sizes = {} + for model, _index in todo: + if model._meta.db_table not in sizes: + sizes[model._meta.db_table] = self._table_bytes(schema_editor, model) + for model, index in sorted(todo, key=lambda item: sizes[item[0]._meta.db_table]): + self._add(schema_editor, model, index) + + @staticmethod + def _table_bytes(schema_editor, model): + with schema_editor.connection.cursor() as cursor: + cursor.execute("SELECT pg_table_size(%s::regclass)", [schema_editor.quote_name(model._meta.db_table)]) + return cursor.fetchone()[0] def database_backwards(self, app_label, schema_editor, from_state, to_state): concurrently = self._concurrent_allowed(schema_editor) @@ -90,17 +111,32 @@ def _add(self, schema_editor, model, index): return if row is not None: cursor.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {name}") - cursor.execute("SELECT pg_table_size(%s::regclass)", [schema_editor.quote_name(model._meta.db_table)]) - small = cursor.fetchone()[0] <= self.plain_build_max_bytes + small = self._table_bytes(schema_editor, model) <= self.plain_build_max_bytes if small and self._add_plainly(schema_editor, model, index): return with schema_editor.connection.cursor() as cursor: + saved = {} + for setting in ("lock_timeout", "maintenance_work_mem", "max_parallel_maintenance_workers"): + cursor.execute("SELECT current_setting(%s)", [setting]) + saved[setting] = cursor.fetchone()[0] cursor.execute("SELECT set_config('lock_timeout', %s, false)", [self.concurrent_build_lock_timeout]) + # Only ever raised: a session already given more keeps it. + cursor.execute( + "SELECT set_config('maintenance_work_mem', %s, false)" + " WHERE pg_size_bytes(current_setting('maintenance_work_mem')) < pg_size_bytes(%s)", + [self.concurrent_build_maintenance_work_mem, self.concurrent_build_maintenance_work_mem], + ) + cursor.execute( + "SELECT set_config('max_parallel_maintenance_workers', %s, false)" + " WHERE current_setting('max_parallel_maintenance_workers')::int < %s", + [str(self.concurrent_build_parallel_workers), self.concurrent_build_parallel_workers], + ) try: schema_editor.add_index(model, index, concurrently=True) finally: with schema_editor.connection.cursor() as cursor: - cursor.execute("RESET lock_timeout") + for setting, value in saved.items(): + cursor.execute("SELECT set_config(%s, %s, false)", [setting, value]) def _wait_for_other_build(self, cursor, index_name): """Wait, at most concurrent_build_wait_seconds, for another backend's build of this index to end.""" diff --git a/unittests/test_auditlog_history_indexes.py b/unittests/test_auditlog_history_indexes.py index 281455ca773..41a8cfdb373 100644 --- a/unittests/test_auditlog_history_indexes.py +++ b/unittests/test_auditlog_history_indexes.py @@ -128,3 +128,95 @@ def test_waiting_for_another_sessions_build_of_the_index_gives_up_at_its_bound(s self.assertRaisesRegex(RuntimeError, "run the migration again once it ends"), ): operation._wait_for_other_build(cursor=None, index_name="dojo_example_hist") + + +def _session_settings(): + """The settings a concurrent index build changes for its own session.""" + with connection.cursor() as cursor: + cursor.execute( + "SELECT current_setting('lock_timeout'), current_setting('maintenance_work_mem')," + " current_setting('max_parallel_maintenance_workers')", + ) + return cursor.fetchone() + + +class HistoryLookupIndexBuildSpeedTest(DojoTestCase): + + """ + A concurrent build of a history index gets the memory and workers to build fast, and the largest tables go last. + + On a large instance the Finding event table is gigabytes, and the default maintenance_work_mem (64 MB on + managed Postgres) both spills the build's sort to disk and, at 32 MB a participant, leaves Postgres one + parallel worker at most. The upgrade then waited on each of those builds for most of an hour. + """ + + migration_name = "0307_event_history_lookup_indexes" + + def _operation_class(self): + return importlib.import_module(f"dojo.db_migrations.{self.migration_name}").AddHistoryLookupIndexes + + def _record_concurrent_build(self, operation): + """Run ``_add`` down its concurrent path with the build itself replaced; the session settings it ran under.""" + event_model = core.event_models(tracks_model=apps.get_model("dojo", "Finding"))[0] + index = mock.Mock() + index.name = "dojo_test_build_speed_hist" + operation.plain_build_max_bytes = -1 # every table counts as large: build CONCURRENTLY + seen = {} + with connection.schema_editor(atomic=False) as schema_editor: + + def recording_add_index(model, idx, *, concurrently=False): + seen["concurrently"] = concurrently + seen["settings"] = _session_settings() + + with mock.patch.object(schema_editor, "add_index", side_effect=recording_add_index): + operation._add(schema_editor, event_model, index) + return seen + + def test_a_concurrent_build_runs_with_more_memory_and_workers_and_restores_the_session(self): + with connection.cursor() as cursor: + cursor.execute("SET maintenance_work_mem = '64MB'") + cursor.execute("SET max_parallel_maintenance_workers = 2") + before = _session_settings() + + seen = self._record_concurrent_build(self._operation_class()(indexes=[])) + + self.assertEqual({"concurrently": True, "settings": ("1h", "512MB", "4")}, seen) + self.assertEqual(before, _session_settings()) + + def test_a_session_that_already_has_more_keeps_it(self): + with connection.cursor() as cursor: + cursor.execute("SET maintenance_work_mem = '2GB'") + cursor.execute("SET max_parallel_maintenance_workers = 6") + + seen = self._record_concurrent_build(self._operation_class()(indexes=[])) + + self.assertEqual(("1h", "2GB", "6"), seen["settings"]) + + def test_the_largest_tables_are_built_last(self): + operation_class = self._operation_class() + sizes = {"dojo_findingevent": 6 * 1024**3, "dojo_testevent": 11 * 1024**2, "dojo_productevent": 3 * 1024**2} + indexes = [ + ("findingevent", mock.Mock(name="finding_a")), + ("testevent", mock.Mock(name="test_a")), + ("findingevent", mock.Mock(name="finding_b")), + ("productevent", mock.Mock(name="product_a")), + ] + operation = operation_class(indexes=indexes) + to_state = mock.Mock() + to_state.apps.get_model.side_effect = lambda _app, name: mock.Mock(_meta=mock.Mock(db_table=f"dojo_{name}")) + schema_editor = mock.Mock() + schema_editor.connection.vendor = "postgresql" + schema_editor.connection.in_atomic_block = False + built = [] + + with ( + mock.patch.object(operation_class, "allow_migrate_model", return_value=True), + mock.patch.object( + operation_class, "_table_bytes", side_effect=lambda _se, model: sizes[model._meta.db_table], + ), + mock.patch.object(operation_class, "_add", side_effect=lambda _se, _model, index: built.append(index)), + ): + operation.database_forwards("dojo", schema_editor, from_state=None, to_state=to_state) + + # Smallest table first; a table's own indexes keep the migration's order. + self.assertEqual([indexes[3][1], indexes[1][1], indexes[0][1], indexes[2][1]], built) From c19b1bfc5a437bfd7070390ca642c828f5b20713 Mon Sep 17 00:00:00 2001 From: Blake Owens <76979297+blakeaowens@users.noreply.github.com> Date: Thu, 12 Nov 2026 20:24:40 +0000 Subject: [PATCH 10/61] docs(risk-acceptance): say approving a request leaves out findings closed while it waited (#16811) Claude-Session: https://claude.ai/code/session_01HJqD5Cv3VzGrZWHeaGqoU4 Co-authored-by: Claude Opus 5.5 --- .../triage_findings/findings_workflows/PRO__risk_acceptance.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/triage_findings/findings_workflows/PRO__risk_acceptance.md b/docs/content/triage_findings/findings_workflows/PRO__risk_acceptance.md index 4a5fcc39896..1cd513fa4ed 100644 --- a/docs/content/triage_findings/findings_workflows/PRO__risk_acceptance.md +++ b/docs/content/triage_findings/findings_workflows/PRO__risk_acceptance.md @@ -102,7 +102,7 @@ Findings can only be added to a single Risk Acceptance at once. If the **Risk A A Finding belongs to one Risk Acceptance at a time. A Finding that is already Risk Accepted, through another Full Risk Acceptance or a Simple Risk Acceptance, cannot be added to a second one, and with Risk Acceptances 2.0 enabled neither can a Finding that an undecided security exception request already names. The Findings step of **New Risk Acceptance** and **Add Findings** on an existing Risk Acceptance list only Findings that are not Risk Accepted, not duplicates, and not waiting on an exception request. -Approving a security exception request accepts only the Findings that nothing else holds by then. A Finding that was Risk Accepted another way while the request waited, for example by reinstating an expired Risk Acceptance that already listed it, is removed from the request and stays accepted where it is. +Approving a security exception request accepts only the Findings that nothing else holds by then. A Finding that was Risk Accepted another way while the request waited, for example by reinstating an expired Risk Acceptance that already listed it, is removed from the request and stays accepted where it is. A Finding that was closed while the request waited (Mitigated, False Positive or Out of Scope) is removed from the request too and stays closed: approving the request never reopens it or accepts it. Its row in the request's **Acceptance History** is marked Removed, with the reason. The API applies the same rule. `POST /api/v2/risk_acceptance/`, a `PATCH` or `PUT` that adds Findings, and `POST /api/v2/risk_acceptance/request/` return `400` and name each Finding that is already held, for example: From b890aeb598b922fb63a123a6219395db72fb4960 Mon Sep 17 00:00:00 2001 From: Blake Owens <76979297+blakeaowens@users.noreply.github.com> Date: Thu, 12 Nov 2026 20:29:56 +0000 Subject: [PATCH 11/61] docs(triage-engine): say which matching passes the import and reimport previews count (#16809) * docs(triage-engine): say which matching passes the import and reimport previews count Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GLL1Wc7eenQgvZJnGkx7Qe * docs(triage-engine): Global Component is previewed when Locations is off Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GLL1Wc7eenQgvZJnGkx7Qe --------- Co-authored-by: Claude Opus 5.5 --- docs/content/automation/triage_engine/node_reference.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/automation/triage_engine/node_reference.md b/docs/content/automation/triage_engine/node_reference.md index 658aa5b4a73..eaf4d838dd5 100644 --- a/docs/content/automation/triage_engine/node_reference.md +++ b/docs/content/automation/triage_engine/node_reference.md @@ -224,7 +224,7 @@ Runs when deduplication matches a newly imported Finding to an existing one. Use | **When the Match Is in Another Asset** | `Mark as Duplicate` | `Mark as Duplicate` leaves deduplication as it is and runs the rule on the new duplicates. `Keep Both Active` stops deduplication from marking a Finding in scope as a duplicate of a Finding in another Asset. | | **Scope** | empty | Which newly imported Findings this rule considers. Empty means every Finding the rule owner can see. | -With **Keep Both Active**, the decision is made during deduplication, before anything is written: a Finding inside the rule's scope whose only match is in another Asset stays active, keeps its own SLA, and is linked to the Finding it matched. A match inside the Finding's own Asset always wins, so importing the same scan into production a second time makes the new Finding a duplicate of the first production Finding, not another active copy. A Finding that is already linked is not announced again when it is deduplicated again later, and the link keeps it active even after the rule is disabled or deleted. Disabling the rule stops new Findings from being kept active; Findings it already kept stay active and linked until one of the pair is deleted or someone removes the link. Deduplication for every other Finding works as before. The import and reimport previews (`/api/v2/import-scan/preview/` and `/api/v2/reimport-scan/preview/`) match across a Dedupe Pool the way the import does, and count a Finding such a rule would keep active as created, and again under `created_linked`, rather than as a duplicate. The parsed Findings are not saved, so a rule scope that filters on tags, endpoints, locations or other related records matches none of them in the preview, and under the unique-id and legacy algorithms `created_linked` is an upper bound. The rule itself then runs on the Findings that were kept active, so it can copy the matched Finding's status (see **Copy Status from Matched Finding**), tag them, or notify someone. +With **Keep Both Active**, the decision is made during deduplication, before anything is written: a Finding inside the rule's scope whose only match is in another Asset stays active, keeps its own SLA, and is linked to the Finding it matched. A match inside the Finding's own Asset always wins, so importing the same scan into production a second time makes the new Finding a duplicate of the first production Finding, not another active copy. A Finding that is already linked is not announced again when it is deduplicated again later, and the link keeps it active even after the rule is disabled or deleted. Disabling the rule stops new Findings from being kept active; Findings it already kept stay active and linked until one of the pair is deleted or someone removes the link. Deduplication for every other Finding works as before. The import and reimport previews (`/api/v2/import-scan/preview/` and `/api/v2/reimport-scan/preview/`) match across a Dedupe Pool the way the import does, and count a Finding such a rule would keep active as created, and again under `created_linked`, rather than as a duplicate. The parsed Findings are not saved, so a rule scope that filters on tags, endpoints, locations or other related records matches none of them in the preview, and under the unique-id and legacy algorithms `created_linked` is an upper bound. Their counts follow every matching pass the import runs, in its order: the tool's same-tool matching, then the identity-signature match when it is turned on, then cross-tool matching against other tools' Findings. A cross-tool match under the **Global Locations** algorithm, or under **Global Component** with Locations enabled, is not counted, because it compares locations the parsed Findings have not stored yet. The rule itself then runs on the Findings that were kept active, so it can copy the matched Finding's status (see **Copy Status from Matched Finding**), tag them, or notify someone. Each item carries the match on `ctx`: From f9514a4c549b5cd7093e387fb8663e834e4b91b0 Mon Sep 17 00:00:00 2001 From: Blake Owens <76979297+blakeaowens@users.noreply.github.com> Date: Thu, 12 Nov 2026 20:30:32 +0000 Subject: [PATCH 12/61] docs(page-layouts): say a narrowed simple table keeps the height its rows need (#16815) * docs(page-layouts): say a narrowed simple table keeps the height its rows need Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HJqD5Ez7PB1xE6VT3N39Ci * docs(page-layouts): say the row rises with a narrowed simple table, and other widgets keep their height Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HJqD5Ez7PB1xE6VT3N39Ci --------- Co-authored-by: Claude Opus 5.5 --- docs/content/navigation/PRO__page_layouts.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/navigation/PRO__page_layouts.md b/docs/content/navigation/PRO__page_layouts.md index 121537d0a91..4026b6f276b 100644 --- a/docs/content/navigation/PRO__page_layouts.md +++ b/docs/content/navigation/PRO__page_layouts.md @@ -49,7 +49,7 @@ The shipped layouts are read only, so the first time you customize a page Defect In edit mode: - **Move a widget** by dragging it. -- **Resize a widget** by dragging its edge. The grid has twelve columns, and widgets snap to them. +- **Resize a widget** by dragging its edge. The grid has twelve columns, and widgets snap to them. A Simple Table with chosen fields cannot be made shorter than the height those fields need on one line each, and one saved shorter than that is shown at that height, together with the widgets of the same height in its row so the row stays level. A long value or label that wraps in a narrow tile can still need more room. Other widgets keep the height you give them. - **Configure Widget** opens the widget's own settings. What is on offer depends on the widget: which columns a table shows, which fields a panel lists, what a chart plots, and so on. You can also retitle a widget here. - **Remove Widget** takes it off the layout. You can add it back later. - **Add Widget** opens the catalog of widgets available for that page. From 33ca2d601b0d036b89cc604bfe9954c15582d1ae Mon Sep 17 00:00:00 2001 From: Blake Owens <76979297+blakeaowens@users.noreply.github.com> Date: Thu, 12 Nov 2026 20:31:44 +0000 Subject: [PATCH 13/61] docs(login-settings): describe the warning before a session ends (#16817) * docs(login-settings): describe the warning before a session ends Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018B4x1AbucwijBnH3k7TSpF * docs(login-settings): say the session expiry warning is the Pro UI's Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018B4x1AbucwijBnH3k7TSpF --------- Co-authored-by: Claude Opus 5.5 --- docs/content/admin/sso/PRO__login_settings.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/content/admin/sso/PRO__login_settings.md b/docs/content/admin/sso/PRO__login_settings.md index 7d58c8df972..2fd28c4c5dc 100644 --- a/docs/content/admin/sso/PRO__login_settings.md +++ b/docs/content/admin/sso/PRO__login_settings.md @@ -25,6 +25,7 @@ Find it under **Connect > Authorization > Login Settings**. Changing it takes th Session Age and Idle Timeout apply to every login, whether by username and password or by SSO. A user whose session ends, for either reason, is taken to the login page on their next click, and lands back on the page they were on once they sign in again. +- In the Pro UI, a **Session Expiring** warning counts down the time left before a session ends, opening five minutes ahead by default (the `DD_SESSION_EXPIRE_WARNING` environment variable, in seconds, and never more than half the Idle Timeout ahead of an inactivity deadline). While inactivity is what will end the session, **Stay Signed In** starts a new Idle Timeout. When Session Age is what will end it, the warning says the session cannot be extended: save any work, then sign in again. Every open tab shares the session, so staying signed in from one tab closes the warning in the others. - Only requests the user makes count as activity. A page left open in a browser tab still reaches the Idle Timeout, even though it keeps refreshing its notification count in the background. - Activity is recorded at most once a minute, so an idle session can end up to a minute before the Idle Timeout, never after it. - A change to either setting reaches every running server within a minute, and applies to sessions that are already signed in. Shortening Session Age signs out anyone whose login is older than the new value. From 67910760fd5cdd274edb0af2348a4fe1a3903e7f Mon Sep 17 00:00:00 2001 From: Blake Owens <76979297+blakeaowens@users.noreply.github.com> Date: Thu, 12 Nov 2026 20:34:46 +0000 Subject: [PATCH 14/61] docs(insights): describe the test type and test type category filters (#16814) The Insights filter bar narrows every dashboard to findings from chosen test types or test type categories (SCA, DAST, Container, Secrets ...). Claude-Session: https://claude.ai/code/session_018B4x1AmvtKPP7BqHYS8Bwu Co-authored-by: Claude Opus 5.5 --- docs/content/metrics_reports/pro_metrics/PRO__overview.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/content/metrics_reports/pro_metrics/PRO__overview.md b/docs/content/metrics_reports/pro_metrics/PRO__overview.md index 9ba629f80ef..396deb54a27 100644 --- a/docs/content/metrics_reports/pro_metrics/PRO__overview.md +++ b/docs/content/metrics_reports/pro_metrics/PRO__overview.md @@ -20,12 +20,14 @@ Before elaborating on each particular dashboard, there are some commonalities be ### Filtering -All Metrics can be filtered by timeframe, Organization, Asset, and Tag. After adjusting the filter as desired, Apply Filters must be clicked in order for the filter to take effect. If you wish to export a PDF of all charts, tables, and graphs on the dashboard as currently filtered, click Export as PDF. +All Metrics can be filtered by timeframe, Organization, Asset, Tag, Test Type, and Test Type Category. After adjusting the filter as desired, Apply Filters must be clicked in order for the filter to take effect. If you wish to export a PDF of all charts, tables, and graphs on the dashboard as currently filtered, click Export as PDF. The filtering timeframe is limited to the past year, but can otherwise be adjusted to include the past 7, 14, 30, 90, or 180 days. Each dashboard opens filtered to the past 90 days. DefectDojo remembers the timeframe you last applied on any Metrics dashboard, so the next time you open one from the menu it uses that timeframe instead. Clicking Clear Filters is remembered too: dashboards then open with no timeframe filter until you apply a new one. The timeframe is saved in your browser, so it applies per browser rather than following your account to another device. A bookmarked or shared link always keeps the timeframe written in its URL. +The Test Type and Test Type Category filters narrow every chart and table on the dashboard to findings from the chosen scanners. Test Types picks individual test types (for example, a single dependency scanner). Test Type Categories picks whole groups of them, such as SCA, DAST, Container or Secrets, so you can follow dependency debt, runtime risk, base image vulnerabilities or exposed credentials on their own. Each test type belongs to one category: built-in scanners get a default category from their name, and the Category field on the Edit Test Type form sets another one. When both filters are set, a finding has to match both. + Note that filter parameters are rendered in the URL, so you can bookmark multiple pages with various filter parameters. This can be useful for quick reference, or for consistently generating a particular kind of report. ### Submenus From 9286b2c21df0572a027c7903192cfc00ed11d333 Mon Sep 17 00:00:00 2001 From: Blake Owens <76979297+blakeaowens@users.noreply.github.com> Date: Thu, 12 Nov 2026 20:34:51 +0000 Subject: [PATCH 15/61] docs(notifications): say where to set notifications for one asset in the Pro UI (#16812) * docs(notifications): say where to set notifications for one asset in the Pro UI Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GLL1WbzJ8Ut9h66RhUqM1d * docs(notifications): list the Pro events an asset's notification settings offer Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GLL1WbzJ8Ut9h66RhUqM1d --------- Co-authored-by: Claude Opus 5.5 --- .../notifications/configure_personal_notifs.md | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/docs/content/admin/notifications/configure_personal_notifs.md b/docs/content/admin/notifications/configure_personal_notifs.md index 12f00e36d46..b464c80e41c 100644 --- a/docs/content/admin/notifications/configure_personal_notifs.md +++ b/docs/content/admin/notifications/configure_personal_notifs.md @@ -31,9 +31,19 @@ In addition to standard personal notifications, DefectDojo Users can also receiv ![image](images/Configure_System_&_Personal_Notifications_3.png) -This configuration can be changed from the **Notifications** section on the **Asset** page: e.g. `your-instance.defectdojo.com/product/{id}`. +In the Pro UI, open the Asset, then choose **Notifications** from the ⚙️ gear menu in the page header (the same menu appears on each Asset row in the Asset lists). The dialog lists the events that can be followed for one Asset, with a column for each delivery method: -From here, you can set whether you want to receive **🔔 Alert**, **Mail** or **Slack** notifications for actions taken on this particular Asset. These notifications apply in addition to any system\-wide notifications you are already receiving. +* **Engagement Added** and **Engagement Closed** +* **Test Added** and **Scan Added** +* **SLA Breach** and **SLA Breach (Combined)** +* **Risk Acceptance Expiring** +* **Finding Risk Accepted**, **Threat Intel Escalation** and **AI Component Discovered**: the Pro events that are sent about one Asset + +Tick a box to receive that event over that method, then **Submit**. The box under a method's name ticks or clears it for every event. Opening the dialog changes nothing: an Asset starts with every event off until settings are submitted for it. Other Pro events (connector health, work assignments, review requests and the like) are sent to named people or are not about one Asset, so they are set on Personal Notifications only. + +In the Classic UI, this configuration is changed from the **Notifications** section on the **Asset** page: e.g. `your-instance.defectdojo.com/product/{id}`. + +From here, you can set whether you want to receive **🔔 Alert**, **Mail** or **Slack** notifications for actions taken on this particular Asset. These notifications apply in addition to your Personal Notifications and any system\-wide notifications you are already receiving, and they stop if you lose access to the Asset. Microsoft Teams cannot send personal notifications of any kind, so Teams notifications cannot be chosen from this menu. From 280c7431e93719e91791e845b35de41b637f87de Mon Sep 17 00:00:00 2001 From: Blake Owens <76979297+blakeaowens@users.noreply.github.com> Date: Thu, 12 Nov 2026 20:38:34 +0000 Subject: [PATCH 16/61] docs(triage-engine): document the Merge Findings node (#16819) * docs(triage-engine): document the Merge Findings node Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018B4x19GRapP5jCLo2KZaTk * docs(triage-engine): say Merge Findings groups the findings that reach it Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018B4x19GRapP5jCLo2KZaTk --------- Co-authored-by: Claude Opus 5.5 --- .../triage_engine/node_reference.md | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/docs/content/automation/triage_engine/node_reference.md b/docs/content/automation/triage_engine/node_reference.md index eaf4d838dd5..5ff46c7d3ed 100644 --- a/docs/content/automation/triage_engine/node_reference.md +++ b/docs/content/automation/triage_engine/node_reference.md @@ -420,6 +420,31 @@ The items themselves are not changed, and pass through unchanged. A rule never r A Finding the rule runs on is never also a target. So **Run Now** of a rule ending in this node over a scope that covers both Assets carries nothing between two linked Findings that are both in the run. Scope such a run to the Asset whose statuses should be carried, so the Findings in the other Asset are only targets. +### Merge Findings + +`finding.merge` + +Merges the active Findings that reach the node and share a key, per Asset, into one Finding, the same merge the Findings list's **Merge** action and `POST /api/v2/findings/merge/` make. The Finding that is kept gets a note listing the Findings merged into it. With **Set Inactive**, each merged Finding also gets a note naming the one it was merged into. + +Only the Findings that reach the node are grouped. A rule on **On Finding Event** (for example after an import) merges that event's Findings among themselves, never into the Asset's existing Findings. To merge an Asset's existing Findings, sweep it with **Manual Run** or **On a Schedule**. + +| Setting | Default | Notes | +|---------|---------|-------| +| **Group By** | Hash Code | Findings that reach the node from one Asset with the same value here are merged: Hash Code, Title, Vulnerability ID or Unique ID From Tool. A Finding with no value is left alone. | +| **Keep** | Oldest Finding | Which Finding of each group the others merge into: Oldest Finding, Newest Finding, or Highest Severity (the oldest among equals). | +| **Add Endpoints** | on | Copy the merged Findings' Endpoints (or Locations) onto the Finding that is kept. | +| **Append Descriptions** | off | Append the merged Findings' descriptions. | +| **Append References** | off | Append the merged Findings' references. | +| **Append Vulnerability IDs** | on | Add the merged Findings' vulnerability IDs. | +| **Append Tags** | off | Add the merged Findings' tags. | +| **Merged Findings** | Set Inactive | Set Inactive, or Delete Permanently. Delete Permanently is offered only while the **Archive** feature flag is on (see [Archive](/asset_modelling/pro__archive/)), needs the rule owner to be allowed to delete those Findings, and asks you to confirm. | + +Findings of different Assets are never merged, and neither are inactive Findings. A merged Finding is inactive afterwards, so running the rule again finds nothing more to merge. + +The rule owner needs edit permission on every Finding a merge touches. A Finding the owner can only see is neither kept nor merged, and the run trace counts it as not edited. With **Delete Permanently**, a merged Finding the owner may edit but not delete is left in place, and the run trace counts it as not deleted. + +The run trace lists which Findings merged into which (the first 50 merges, with a count of the rest), so a dry run shows the merges before you enable the rule. A sweep that runs over many Findings works through them in chunks, and a group can arrive over several of them: the Finding kept is still the one **Keep** names across the whole sweep. Up to 100,000 groups are followed from chunk to chunk this way; past that, a group first reached in a later chunk merges within its chunk, and the run trace counts those groups. + ### Add Tags `finding.add_tags` From 6394b4881eabcd0903cac2b73d67a4a76f4063a0 Mon Sep 17 00:00:00 2001 From: Blake Owens <76979297+blakeaowens@users.noreply.github.com> Date: Thu, 12 Nov 2026 20:49:30 +0000 Subject: [PATCH 17/61] docs(custom-fields): document computed custom fields and rule formulas (#16759) * docs(custom-fields): document computed custom fields and rule formulas Computed custom fields: what a formula reads (built-in numbers, number custom fields, finding counts and aggregates), computed fields reading each other, when values recompute, and the Triage Engine event a moved value raises. Triage Engine: the Findings Changed asset trigger, the Write To target on the custom field nodes, and formulas over finding counts. Co-Authored-By: Claude Opus 5.5 * docs(custom-fields): scoped recompute, the compute confirmation and the sweep lock Co-Authored-By: Claude Opus 5.5 * docs(custom-fields): empty results, recompute triggers and rule chains Describe what the review changed: a record the formula has no result for is emptied, a label-only edit recomputes nothing, the row offers Insert Number too, and a recompute a rule set off stays in that rule's chain. Co-Authored-By: Claude Opus 5.5 * docs(custom-fields): a computed field with no result is empty, and a new formula announces no Finding updates Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- .../asset_modelling/PRO__custom_fields.md | 98 +++++++++++++++++++ .../triage_engine/node_reference.md | 49 ++++++++-- 2 files changed, 141 insertions(+), 6 deletions(-) diff --git a/docs/content/asset_modelling/PRO__custom_fields.md b/docs/content/asset_modelling/PRO__custom_fields.md index f538bf9330b..9c0d98684ca 100644 --- a/docs/content/asset_modelling/PRO__custom_fields.md +++ b/docs/content/asset_modelling/PRO__custom_fields.md @@ -46,6 +46,8 @@ add the fields that apply to it. Each definition has: required field always appears on those forms and cannot be hidden from them. - **Allow Other** (Single Select and Multi Select only): whether the field also offers an **Other** choice that takes a typed value. See [The Other choice](#the-other-choice). +- **Formula** (Integer and Decimal fields): fills the value in from the record's numbers instead of + by hand. See [Computed fields](#computed-fields). ### Data types @@ -87,6 +89,102 @@ typed text to the field's options, which makes it an ordinary option. It is also enabled rule sets the field to a value outside its options, and the message names those rules: update or disable them first. +## Computed fields + +Any Integer or Decimal custom field can be **computed**: DefectDojo fills its value in with a +formula, and nobody types it. A typical use is a health score for each Asset, or a weighted risk on +each Finding. Give the field a formula when you add it, or type one into the **Options or Formula** +column of its row; clearing the formula makes the field an ordinary one again. Because a formula +replaces every value already entered in the field, DefectDojo asks before an existing field becomes +computed, and again before a computed field goes back to being entered by hand (its values stay as +last computed). **Insert Number** under the formula, in the field's row and when you add a field, +lists every number it can read and adds the one you pick to the end of the formula. + +A computed field holds what its formula gives and nothing else. Where the formula has no result for a +record (an input it reads is missing, it divides by zero, it averages no Findings, or a step is too +large to be a number), the record's value is empty, including a value typed in before the field became +computed. + +A formula reads the record's own numbers: + +- **Built-in numbers**: `fields.`. On a Finding: `cwe`, `cvssv3_score`, `cvssv4_score`, + `epss_score`, `epss_percentile`, `line`, `occurrences`, `scanner_confidence`, `priority`, + `risk_score`, `threat_score`, `threat_ladder_rung`, `pain_rating` and `reachability_confidence`. On + an Asset: `grade`, `user_records`, `revenue`, `exposure_confidence` and `patch_cycle_days`. On a + Test: `percent_complete`. +- **Number custom fields**: `custom_fields.` for the record's Integer and Decimal fields, + computed ones included. +- **Finding counts** (on an Organization, Asset, Engagement, Test or Risk Acceptance): + `findings.active.total`, `findings.active.risk.urgent` (and `needs_action`, `medium`, `low`, + `unscored`), `findings.active.severity.critical` (and `high`, `medium`, `low`, `info`), and the + same under `findings.open` for open Findings (active, and not a duplicate, false positive, out of + scope, risk accepted or mitigated). +- **Finding aggregates** (on the same records): `findings...` followed + by any Finding number, built-in or custom. For example `findings.open.max.fields.epss_score` is the + highest EPSS score among the record's open Findings, and `findings.active.sum.custom_fields.cost` + adds up a Finding custom field. A sum over no Findings is 0; an average, lowest or highest over none + is missing. + +It combines them with `+ - * /`, parentheses, comparisons, `and`, `or`, `not`, and the functions +`min`, `max`, `abs`, `round`, `div(a, b, if_zero)`, `coalesce` and `if(test, a, b)`. For example, a +score from 100 down that weighs each active Finding by its priority band: + +``` +100 * (1 - div(10 * findings.active.risk.urgent + 5 * findings.active.risk.needs_action + + 2 * findings.active.risk.medium + findings.active.risk.low, 10 * findings.active.total, 0)) +``` + +### Computed fields that read each other + +A computed field can read another computed field, on the same record or, through an aggregate, on its +Findings: a computed `weighted_risk` on each Finding can feed an Asset's +`findings.active.sum.custom_fields.weighted_risk`. DefectDojo recomputes them in order, so a field is +always written before anything that reads it. A formula that would make fields read each other in a +loop is refused when it is saved, and the message names the loop. A field that a computed formula +reads cannot be deleted, or changed to a type other than Integer or Decimal, until those formulas +stop reading it; the message names them. + +### When values are recomputed + +- **After Findings change.** Saving or deleting a Finding (including an import) recomputes, shortly + afterwards, the Finding's own computed fields and every computed field on its Asset, Organization, + Engagements, Tests and the Risk Acceptances on its Engagements. +- **After an input changes.** Changing a custom field value that a computed formula reads (by hand, + through the API or from a rule) recomputes that record, and a Finding's value recomputes its Asset. + Saving an Asset or a Test recomputes it when a formula reads its built-in numbers. +- **When the formula changes** (or the field's type, or whether it is computed), the field and every + computed field that reads it are recomputed for every record at once. Changing only its label + recomputes nothing. +- **Every hour**, every computed field is recomputed in full. That catches changes made without saving + each record, such as bulk updates, priority recalculations and records moving between Assets. A + sweep still running when the next hour comes is never overlapped; the next one is skipped. + +When a computed field gets a new value, the Triage Engine hears it the same way as a rule writing that +field. On an Asset, a rule triggered [On Asset Event](/automation/triage_engine/node_reference/#on-asset-event) +for updates sees `custom_fields.` in `ctx.changed_fields`, so an If / Filter node with +`ctx.changed_fields` `contains` `custom_fields.` runs only when that value moves. On a Finding, +it is a Finding update. A recompute that leaves the value unchanged raises no event. The one +recompute that announces no Finding updates is the one after the formula itself changes: a new formula +can move the value on every Finding at once, so a rule reading it sees the new value on that Finding's +next change instead. Asset updates are announced as usual. + +When a rule's write is what moved the computed value (the rule set a field the formula reads), the +event belongs to that rule's run, as a change the rule made itself would: it does not start the same +rule again, and it counts toward the limit on how many rules one change can set off in a chain. A rule +that writes a field from a computed field reading that same field therefore runs once per change, +rather than feeding itself forever. + +Counts and aggregates include every Finding of the record, whoever is looking, so everyone sees the +same value, as with the Asset grade. An Integer field rounds half away from zero; a Decimal field keeps +up to four decimal places. A record whose formula reads a number it does not have, or divides by zero +with `/`, is left empty: use `div` or `coalesce` to give such a record a value of its own. + +A computed value cannot be typed on the record, set through the API, or written by a rule; on the +record and its forms it shows read-only, with its formula. A field cannot be made computed while an +enabled rule writes it (the message names the rules). To compute a value only on some records, or on a +schedule of your own, use a [rule with a formula](/automation/triage_engine/node_reference/#formulas) +instead. + ## Filling in values Once fields are defined for a record type, fill them in from the record's **⋮ (kebab) menu → Custom diff --git a/docs/content/automation/triage_engine/node_reference.md b/docs/content/automation/triage_engine/node_reference.md index 5ff46c7d3ed..ecb67def8e4 100644 --- a/docs/content/automation/triage_engine/node_reference.md +++ b/docs/content/automation/triage_engine/node_reference.md @@ -42,9 +42,11 @@ Runs when Assets (Products) are created or updated. Tag changes count as updates | Setting | Default | Notes | |---------|---------|-------| -| **Event** | `created` | Which Asset change wakes this rule: `created`, `updated`, or `any` for both. | +| **Event** | `created` | Which Asset change wakes this rule: `created`, `updated`, `any` for both, or `findings_changed`. | | **Scope** | empty | Which Assets this rule considers, in the Assets list's own filters. Empty means every Asset the rule owner can see. | +**Findings Changed** (`findings_changed`) wakes the rule when a Finding of the Asset is created, closed, reopened or updated, including a change another rule made. It is not part of `any`, so a rule written for any change to the Asset itself does not start waking on every Finding save. It is what keeps a score computed from the Asset's Findings current (see [Formulas](#formulas)). Changes written in bulk without saving each Finding, and a Finding moving to another priority band, do not wake it, so pair such a rule with a scheduled copy, or use a [computed custom field](/asset_modelling/pro__custom_fields/#computed-fields), which also recomputes hourly. + Assets named by the event are matched against scope before they enter the graph. For an update, the names of the changed fields travel with the items as `ctx.changed_fields`, so an If / Filter node can say "only when the organization changed" (`ctx.changed_fields` `contains` `organization`) or "only when tagged" (`contains` `tags`). ### On a Schedule @@ -620,8 +622,9 @@ Writes one of this instance's [Custom Fields](/asset_modelling/pro__custom_field | Setting | Notes | |---------|-------| -| **Field** | Which custom field to write. | -| **Value** | Typed to the field: text takes a template with `{{finding.title}}` style placeholders, numbers take a number, dates take a `YYYY-MM-DD` date, booleans take true or false, and the select types offer the field's own options. | +| **Write To** | `This Finding` by default, or a record the Finding belongs to: its Test, Engagement, Asset or Organization. | +| **Field** | Which custom field to write, from the fields defined for the record chosen in **Write To**. | +| **Value** | Typed to the field: text takes a template with `{{finding.title}}` style placeholders, Integer and Decimal fields take a number or a [formula](#formulas), dates take a `YYYY-MM-DD` date, booleans take true or false, and the select types offer the field's own options. | The value is checked against the field's current definition when the rule is saved and again on every run, so a rule can never write a value the field's data type refuses. A text template that renders empty for a Finding leaves that Finding untouched (removal is the Clear node's job), setting a multi-select replaces the whole stored list, and Findings already holding the value are left alone. @@ -630,6 +633,7 @@ Three behaviors worth knowing: * **Scope is the boundary.** Like every Findings node, the write applies to every Finding the trigger produced under the rule owner's visibility. * **A custom field write is not a Finding save.** Nothing that follows a Finding save runs: no SLA recomputation, no deduplication, no re-prioritization. A custom field edited by hand on a Finding's page does not wake **On Finding Event** rules either. A write made by a rule does cascade: other rules see it as an `updated` event, and later nodes in the same run read the new value. * **The field must still exist.** If the field is deleted after the rule was saved, the run errors naming it. If Custom Fields is switched off entirely, the node skips with the reason on its trace instead, so a saved rule never starts erroring over a feature flag. +* **Writing to a record the Finding belongs to.** Many Findings share one Test, Engagement, Asset or Organization, so that record is written once, from the first of its Findings in the run, and its change appears once on that Finding's provenance timeline. The rule owner needs edit on the record itself; records they cannot edit are counted as `skipped_unauthorized`. A [computed](/asset_modelling/pro__custom_fields/#computed-fields) field is never offered: the instance fills it in. ### Clear a Custom Field @@ -639,8 +643,39 @@ Removes one custom field's value from the Finding. The value it held is recorded | Setting | Notes | |---------|-------| +| **Write To** | `This Finding`, or its Test, Engagement, Asset or Organization, as on **Set a Custom Field**. | | **Field** | Which custom field to clear. | +### Formulas + +An Integer or Decimal custom field's value can be a formula over the numbers the item carries, evaluated per item when the rule runs. A plain number is a formula too, so a rule saved with a fixed number reads the same. For example, a health score that weighs each active Finding by its priority band: + +``` +100 * (1 - div(10 * product.findings.active.risk.urgent + 5 * product.findings.active.risk.needs_action + + 2 * product.findings.active.risk.medium + product.findings.active.risk.low, + 10 * product.findings.active.total, 0)) +``` + +* **Numbers.** A formula reads a path, written bare or in `{{ }}`: the item's number fields, its number custom fields, and the Finding counts below. **Insert Field** beside the formula lists exactly the paths a formula can read. +* **Operators.** `+ - * /` and parentheses; the comparisons `< <= > >= == !=`, `and`, `or` and `not`, which give 1 or 0. +* **Functions.** `min`, `max`, `abs`, `round(x)` or `round(x, places)`, `div(a, b, if_zero)` (`a / b`, or `if_zero` when `b` is 0), `coalesce(a, b, ...)` (the first that has a value) and `if(test, a, b)`. +* **Results.** An Integer field rounds half away from zero; a Decimal field keeps up to four decimal places. +* **Skipped items.** An item whose formula reads a number the item does not have, divides by zero with `/`, or overflows is skipped: nothing is written, the field keeps its previous value, and the node trace counts it as `skipped_missing_input`, `skipped_division_by_zero` or `skipped_not_finite`. +* **Writing to a parent record.** A formula writing a Test's, Engagement's, Asset's or Organization's field may read only that record and the records above it, so every Finding of the record computes the same value. The save refuses a formula that reads anything else. + +#### Finding counts + +The Test, Engagement, Asset and Organization of an item each carry `findings`, the counts of the Findings beneath them that the rule owner can see. An Asset graph's items carry the Asset's and the Organization's. Each count is a number in conditions and templates too. + +| Path | Counts | +|------|--------| +| `.findings.active.total` | Active Findings (the number the lists' Active Findings column shows). | +| `.findings.active.risk.` | Active Findings in a priority band: `urgent`, `needs_action`, `medium`, `low`, or `unscored` for Findings not yet prioritized. | +| `.findings.active.severity.` | Active Findings of a severity: `critical`, `high`, `medium`, `low`, `info`. | +| `.findings.open.*` | The same counts over open Findings: active, and not a duplicate, false positive, out of scope, risk accepted or mitigated. | + +`` is `test`, `engagement`, `product` (or `asset`) or `product_type` (or `organization`). + ## Assets These nodes change Assets (Products). Like the Findings nodes, every change is attributed back to the rule, run and node that made it, and shows up on the Asset's provenance. They only join graphs whose trigger produces Assets. @@ -738,8 +773,9 @@ Writes one of this instance's [Custom Fields](/asset_modelling/pro__custom_field | Setting | Notes | |---------|-------| -| **Field** | Which custom field to write. One entry per Asset custom field defined on the instance. | -| **Value** | Typed to the field. | +| **Write To** | `This Asset` by default, or its Organization. | +| **Field** | Which custom field to write. One entry per custom field defined for the record chosen in **Write To**. | +| **Value** | Typed to the field. Integer and Decimal fields take a number or a [formula](#formulas) over the Asset's numbers and Finding counts. | Two behaviors of its own: Assets the rule owner may not edit are counted on the node trace as `skipped_unauthorized` rather than touched, and the write lands on the custom field value rather than the Asset row itself, so nothing that follows an Asset edit runs. A custom field edited by hand does not wake **On Asset Event** rules; a write made by a rule still cascades as an `updated` event. @@ -747,10 +783,11 @@ Two behaviors of its own: Assets the rule owner may not edit are counted on the `asset.clear_custom_field` -Removes one custom field's value from the Asset, recording what it held on the Asset's provenance. +Removes one custom field's value from the Asset, or from its Organization, recording what it held on the Asset's provenance. | Setting | Notes | |---------|-------| +| **Write To** | `This Asset`, or its Organization. | | **Field** | Which custom field to clear. | ### Naming a configuration or engine the rule owner cannot see From 9f3a2bfc7597b809f8224ca1b3cb8f5cebebd8af Mon Sep 17 00:00:00 2001 From: Cody Maffucci <46459665+Maffooch@users.noreply.github.com> Date: Thu, 12 Nov 2026 21:45:42 +0000 Subject: [PATCH 18/61] fix(importers): do not load already-closed findings for the reimport close pass (#16798) Reimport loaded every finding of the to-mitigate set as a full instance, with its JIRA relations, before close_old_findings() looked at it. That set is every finding of the test the report no longer carries, closed ones included, and the chunked load appended every chunk to one list. On a test whose findings are nearly all closed, a reimport that closed nothing held all of them in memory at once, about 8 KiB each, and ran the worker out of memory on tests with a few hundred thousand closed findings. Reported via a customer support ticket. _hydrate_findings_for_close_old() now loads only the findings close_old_findings() can close (mitigated unset, or is_mitigated false: the same test it already applied) and keeps the other ids in already_closed_to_mitigate. The one thing close_old_findings() still did with an already-closed finding, pushing its finding group to JIRA, now reads those groups by id. Which findings close, reopen and match is unchanged. Co-authored-by: Claude Opus 5.5 --- dojo/importers/default_reimporter.py | 38 +++++++- unittests/test_importers_closeold.py | 133 ++++++++++++++++++++++++++- 2 files changed, 169 insertions(+), 2 deletions(-) diff --git a/dojo/importers/default_reimporter.py b/dojo/importers/default_reimporter.py index bf6c18b1eeb..04e75ddd8a5 100644 --- a/dojo/importers/default_reimporter.py +++ b/dojo/importers/default_reimporter.py @@ -802,7 +802,15 @@ def _hydrate_findings_for_close_old(self, finding_ids: set[int]) -> list[Finding finding shares self.test the same way reimport matching's candidates do (see ProReImporter._reimport_hash_candidate_index's "share self.test" comment): one instance for the whole run instead of one lazy-loaded copy per finding closed. + + A finding that is already closed (mitigated set and is_mitigated) is not loaded: + close_old_findings() leaves it as it is, and on a test whose findings are nearly all + closed it is nearly all of the to-mitigate set, so loading it kept every closed finding + of the test in memory at once. Its id goes to self.already_closed_to_mitigate instead, + which close_old_findings() reads for the one thing it still did with such a finding: + pushing its finding group to JIRA. """ + self.already_closed_to_mitigate = [] if not finding_ids: return [] findings: list[Finding] = [] @@ -810,13 +818,30 @@ def _hydrate_findings_for_close_old(self, finding_ids: set[int]) -> list[Finding # jira_issue and the group's are read for every finding to decide the JIRA push for finding in ( Finding.objects.filter(id__in=chunk) + .filter(Q(mitigated__isnull=True) | Q(is_mitigated=False)) .select_related("jira_issue") .prefetch_related("finding_group_set__jira_issue") ): finding.test = self.test findings.append(finding) + # Ids only: the ones not loaded above, deleted ones included. A deleted finding has no + # finding group, so reading the groups through these ids drops it the way the + # status refresh drops a loaded one. + loaded = {finding.id for finding in findings} + self.already_closed_to_mitigate = [finding_id for finding_id in finding_ids if finding_id not in loaded] return findings + def _finding_groups_of(self, finding_ids: list[int]) -> set[Finding_Group]: + """The finding group of each of these findings, read without loading the findings.""" + groups: set[Finding_Group] = set() + for chunk in batched(finding_ids, _CLOSE_OLD_FINDINGS_STATUS_FIELDS_CHUNK, strict=False): + groups.update( + Finding_Group.objects.filter(findings__in=Finding.objects.filter(id__in=chunk)) + .select_related("jira_issue") + .distinct(), + ) + return groups + def _sync_close_old_finding_status_fields(self, findings: list[Finding]) -> list[Finding]: """ Refresh false_p, risk_accepted, and out_of_scope from the DB for each finding. @@ -874,7 +899,16 @@ def close_old_findings( """ Updates the status of findings that were detected as "old" by the reimport process findings methods + + The already-closed findings _hydrate_findings_for_close_old() left out are still + here as ids (self.already_closed_to_mitigate): there is nothing to close on them, but + their finding groups are pushed to JIRA with the rest, as they were when every + to-mitigate finding was loaded. A subclass that drops findings from ``findings`` + before calling this drops their ids from that list too. """ + already_closed_ids = getattr(self, "already_closed_to_mitigate", None) or [] + # Read once: the next reimport through this instance hydrates its own + self.already_closed_to_mitigate = [] # First check if close old findings is desired if self.close_old_findings_toggle is False: return [] @@ -905,7 +939,9 @@ def close_old_findings( # We dont check if the finding jira sync is applicable quite yet until we can get in the loop # but this is a way to at least make it that far if self.findings_groups_enabled and (self.push_to_jira or getattr(self.jira_instance, "finding_jira_sync", False)): - for finding_group in {finding.finding_group for finding in findings if finding.finding_group is not None}: + finding_groups = {finding.finding_group for finding in findings if finding.finding_group is not None} + finding_groups |= self._finding_groups_of(already_closed_ids) + for finding_group in finding_groups: # Check the push_to_jira flag again to potentially shorty circuit without checking for existing findings if self.push_to_jira or jira_services.is_keep_in_sync(finding_group, prefetched_jira_instance=self.jira_instance): jira_services.push(finding_group) diff --git a/unittests/test_importers_closeold.py b/unittests/test_importers_closeold.py index d709cae0ebd..cda111615b4 100644 --- a/unittests/test_importers_closeold.py +++ b/unittests/test_importers_closeold.py @@ -6,7 +6,15 @@ import dojo.risk_acceptance.helper as ra_helper from dojo.importers.default_importer import DefaultImporter from dojo.importers.default_reimporter import DefaultReImporter -from dojo.models import Development_Environment, Engagement, Finding, Product, Product_Type, User +from dojo.models import ( + Development_Environment, + Engagement, + Finding, + Finding_Group, + Product, + Product_Type, + User, +) from .dojo_test_case import DojoTestCase, get_unit_tests_scans_path @@ -391,3 +399,126 @@ def test_close_old_findings_skips_unsaved_finding(self): {finding.pk for finding in closed_findings}, ) self.assertIsNone(unsaved_finding.pk, msg="close_old_findings must not save a new finding") + + +class TestReimportCloseOldSkipsAlreadyClosedFindings(DojoTestCase): + + """ + Regression: reimport hydrated every to-mitigate finding, closed ones included. + + The to-mitigate set is every finding of the test the report no longer carries, so on a + test whose findings are almost all closed it is almost all closed findings. Each one was + loaded as a full instance (with its JIRA relations) only for close_old_findings() to drop + it again as already closed, and a test with a few hundred thousand closed findings ran + the worker out of memory on a reimport that closed nothing. + """ + + def setUp(self): + super().setUp() + self.scan_type = "Acunetix Scan" + self.user, _ = User.objects.get_or_create(username="admin") + product_type, _ = Product_Type.objects.get_or_create(name="closeold_already_closed") + self.environment, _ = Development_Environment.objects.get_or_create(name="Development") + product, _ = Product.objects.get_or_create( + name="TestReimportCloseOldAlreadyClosed", + description="Test", + prod_type=product_type, + ) + self.engagement, _ = Engagement.objects.get_or_create( + name="Close Old Already Closed Findings", + product=product, + target_start=timezone.now(), + target_end=timezone.now(), + ) + with (get_unit_tests_scans_path("acunetix") / "acunetix360_many_findings.json").open(encoding="utf-8") as scan: + importer = DefaultImporter( + close_old_findings=False, + user=self.user, + lead=self.user, + scan_date=None, + environment=self.environment, + active=True, + verified=False, + engagement=self.engagement, + scan_type=self.scan_type, + ) + self.test, _, len_new_findings, _, _, _, _ = importer.process_scan(scan) + self.assertGreaterEqual(len_new_findings, 6) + + pks = sorted(self.test.finding_set.values_list("pk", flat=True)) + self.closed_at = timezone.now() - timezone.timedelta(days=30) + # Closed the way a reimport or a person closes a finding: both flags set + self.already_closed_pks = set(pks[:-3]) + Finding.objects.filter(pk__in=self.already_closed_pks).update( + active=False, is_mitigated=True, mitigated=self.closed_at, + ) + # Half-closed rows close_old_findings still closes (it requires both flags to skip one) + self.flagged_without_date_pk = pks[-3] + Finding.objects.filter(pk=self.flagged_without_date_pk).update(active=False, is_mitigated=True, mitigated=None) + self.dated_without_flag_pk = pks[-2] + Finding.objects.filter(pk=self.dated_without_flag_pk).update(is_mitigated=False, mitigated=self.closed_at) + self.open_pks = {self.flagged_without_date_pk, self.dated_without_flag_pk, pks[-1]} + + def _reimporter(self, **kwargs): + return DefaultReImporter( + test=self.test, + user=self.user, + lead=self.user, + scan_date=None, + environment=self.environment, + active=True, + verified=False, + scan_type=self.scan_type, + close_old_findings=True, + **kwargs, + ) + + def test_reimport_hydrates_only_findings_it_can_close(self): + hydrated = [] + hydrate = DefaultReImporter._hydrate_findings_for_close_old + + def spy(importer, finding_ids): + findings = hydrate(importer, finding_ids) + hydrated.extend(findings) + return findings + + with ( + mock.patch.object(DefaultReImporter, "_hydrate_findings_for_close_old", spy), + (get_unit_tests_scans_path("acunetix") / "acunetix360_zero_finding.json").open(encoding="utf-8") as scan, + ): + _, _, len_new_findings, len_closed_findings, _, _, _ = self._reimporter().process_scan(scan) + + self.assertEqual(0, len_new_findings) + self.assertEqual( + self.open_pks, {finding.pk for finding in hydrated}, + msg="only the to-mitigate findings that are not already closed may be loaded as instances", + ) + # Same outcome as before: the open ones close, the closed ones are left as they were + self.assertEqual(len(self.open_pks), len_closed_findings) + for finding in Finding.objects.filter(pk__in=self.open_pks): + self.assertTrue(finding.is_mitigated, msg=f"finding {finding.pk} should be closed") + self.assertIsNotNone(finding.mitigated, msg=f"finding {finding.pk} should carry a mitigated date") + for finding in Finding.objects.filter(pk__in=self.already_closed_pks): + self.assertEqual(self.closed_at, finding.mitigated, msg=f"finding {finding.pk} was closed again") + self.assertFalse(finding.notes.exists(), msg=f"finding {finding.pk} got a mitigation note") + + def test_close_old_still_pushes_the_groups_of_already_closed_findings(self): + """The finding-group JIRA push covers every to-mitigate finding's group, closed or not.""" + closed_pk = min(self.already_closed_pks) + open_pk = max(self.open_pks) + closed_group = Finding_Group.objects.create(name="closed group", test=self.test, creator=self.user) + closed_group.findings.add(closed_pk) + open_group = Finding_Group.objects.create(name="open group", test=self.test, creator=self.user) + open_group.findings.add(open_pk) + importer = self._reimporter(push_to_jira=True) + importer.findings_groups_enabled = True + + with mock.patch("dojo.importers.default_reimporter.jira_services.push") as push: + closed = importer.close_old_findings(importer._hydrate_findings_for_close_old({closed_pk, open_pk})) + + self.assertEqual([open_pk], [finding.pk for finding in closed]) + self.assertEqual( + {closed_group.pk, open_group.pk}, + {call.args[0].pk for call in push.call_args_list}, + ) + self.assertEqual(2, push.call_count, msg="each group is pushed once") From a316dc452bb35b8031a76fee0035b3e85bc04902 Mon Sep 17 00:00:00 2001 From: Cody Maffucci <46459665+Maffooch@users.noreply.github.com> Date: Thu, 12 Nov 2026 21:45:45 +0000 Subject: [PATCH 19/61] fix(locations): new importer references follow the finding's status (#16800) * fix(locations): new importer references follow the finding's status An import created every new location finding reference as Active, even for a finding the report already marks mitigated or inactive, and never reactivated an Asset reference that was Mitigated when a new Active finding reference joined it. The Locations list then counted closed findings as active on a location whose Asset reference read Mitigated. - New finding references take the status the UI add path already uses (status_from_finding, now a pure function in dojo.location.status). - A new Asset reference is Mitigated when every new finding reference to its location is closed; a location recorded for the product alone stays Active. - An existing Mitigated Asset reference becomes Active again when a new Active finding reference joins it (one UPDATE, only when that happens; the existing-reference query now also reads the status, so no query is added). Co-Authored-By: Claude Opus 5.5 * docs(locations): describe how references follow a finding's status An import now gives a new Finding Reference its finding's status, and a finding closed or reopened outside an import brings its references and the Location's Asset Reference along. The Locations overview said every new reference starts Active. Co-Authored-By: Claude Opus 5.5 * fix(locations): keep Active refs for inactive imported findings The importer gave every new finding reference its finding's full status, so an inactive or duplicate finding got a Mitigated reference. That changed the import contract test_import_reimport pins (findings imported with active=False keep Active references). A new finding reference is now Mitigated only when the report marks the finding mitigated (is_mitigated, excluding False Positive and Out Of Scope, which the save also marks mitigated, and Risk Accepted). Every other finding keeps the Active reference imports have always created. The Asset reference rule follows: Mitigated only when every new finding reference to its location is Mitigated. Location.status_from_finding is back to its dev form, since the importer no longer uses it. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- .../locations/PRO__locations_overview.md | 4 +- dojo/importers/location_manager.py | 83 ++++++++++---- ...est_location_reference_status_on_create.py | 104 ++++++++++++++++++ 3 files changed, 168 insertions(+), 23 deletions(-) create mode 100644 unittests/test_location_reference_status_on_create.py diff --git a/docs/content/asset_modelling/locations/PRO__locations_overview.md b/docs/content/asset_modelling/locations/PRO__locations_overview.md index e4f3fd2562f..fb4ad854b22 100644 --- a/docs/content/asset_modelling/locations/PRO__locations_overview.md +++ b/docs/content/asset_modelling/locations/PRO__locations_overview.md @@ -60,6 +60,8 @@ To show one status for a Location on a Finding, DefectDojo combines its referenc 2. Otherwise the Location is `Active` if any Test still reports it. 3. Otherwise it is `Mitigated`. +Closing a Finding outside an import (an edit, the Close action or a bulk edit) sets its `Active` references to `Mitigated`. Reopening it sets its `Mitigated` references back to `Active`, except one a later scan replaced. `False Positive`, `Risk Accepted` and `Out of Scope` references keep their status. An Asset Reference is `Active` while any Finding on that Asset still has an `Active` reference to the Location, and `Mitigated` otherwise. + When you upgrade, every existing reference is assigned to its Finding's Test. A reference added through the API or the UI belongs to the Finding's own Test. This separation is what allows a library to exist on an Asset *without* needing a Finding — a missing capability in the old Components model. @@ -69,7 +71,7 @@ This separation is what allows a library to exist on an Asset *without* needing When a parser produces a Finding that references a URL or library, the importer: 1. Looks up an existing Location matching the URL or pURL; if none exists, it creates one. -2. Creates a Finding Reference linking the Finding to the Location with status `Active`. +2. Creates a Finding Reference linking the Finding to the Location. The reference is `Mitigated` when the report already marks the Finding mitigated, and `Active` otherwise (including an inactive, duplicate, False Positive, Out of Scope or Risk Accepted Finding). 3. Creates (or reuses) an Asset Reference so the Location also lives on the parent Asset. Existing parsers have been updated to emit Location data when the feature flag is on, and to fall back to the legacy Endpoint model when it is off. No reconfiguration is needed when Locations are enabled — the next import will route through the Locations pipeline automatically. diff --git a/dojo/importers/location_manager.py b/dojo/importers/location_manager.py index 848561d47f8..cdaa02a7dd5 100644 --- a/dojo/importers/location_manager.py +++ b/dojo/importers/location_manager.py @@ -35,6 +35,19 @@ class StatusUpdateEntry(NamedTuple): user: Dojo_User +def new_finding_ref_status(finding: Finding) -> str: + """ + The status of a finding ref an import creates: Mitigated when the report marks the finding mitigated. + + Every other finding (inactive, unverified, duplicate, false positive, out of scope, risk accepted) gets an Active + ref, as imports always have. A false positive or out of scope finding is saved with is_mitigated set, so those + are excluded explicitly. + """ + if finding.is_mitigated and not (finding.false_p or finding.out_of_scope or finding.risk_accepted): + return FindingLocationStatus.Mitigated + return FindingLocationStatus.Active + + class LocationManager(BaseLocationManager): def __init__(self, product: Product) -> None: @@ -191,26 +204,8 @@ def _persist_locations(self) -> None: # List of all location IDs, for querying existing refs all_location_ids = [loc.location_id for loc in saved] - # Determine necessary product refs to create - existing_product_refs: set[int] = set( - LocationProductReference.objects.filter( - location_id__in=all_location_ids, - product=self._product, - ).values_list("location_id", flat=True), - ) - for location in saved: - if location.location_id not in existing_product_refs: - assoc = location.get_association_data() - all_product_refs.append(LocationProductReference( - location_id=location.location_id, - product=self._product, - status=ProductLocationStatus.Active, - relationship=assoc.relationship_type, - relationship_data=assoc.relationship_data, - )) - existing_product_refs.add(location.location_id) - - # Determine necessary finding refs to create + # Determine necessary finding refs to create. A finding the report already marks mitigated gets a Mitigated + # ref, so it does not leave an Active ref behind (see new_finding_ref_status). if self._locations_by_finding: all_finding_ids = [finding.id for finding, _ in self._locations_by_finding] # Strictly speaking this returns more rows than we need (it's the cross of the location/finding lists rather @@ -225,6 +220,7 @@ def _persist_locations(self) -> None: for finding, cleaned_locations in self._locations_by_finding: test_id = self.attribution_test_id(finding) + status = new_finding_ref_status(finding) # Locations were already cleaned at record time — identity_hash is set, so we can # look up the persisted location directly. for location in cleaned_locations: @@ -236,14 +232,51 @@ def _persist_locations(self) -> None: location_id=saved_loc.location_id, finding=finding, test_id=test_id, - status=FindingLocationStatus.Active, + status=status, relationship=assoc.relationship_type, relationship_data=assoc.relationship_data, )) existing_finding_ref_keys.add(finding_ref_key) + all_finding_refs = self._drop_finding_refs_for_missing_findings(all_finding_refs) + + # Determine necessary product refs to create. A product ref is Active unless every new finding ref to its + # location is Mitigated; a location recorded for the product alone stays Active, as before. + new_ref_statuses: dict[int, set[str]] = {} + for ref in all_finding_refs: + new_ref_statuses.setdefault(ref.location_id, set()).add(ref.status) + newly_active = { + location_id for location_id, statuses in new_ref_statuses.items() + if FindingLocationStatus.Active in statuses + } + existing_product_refs: dict[int, str] = dict( + LocationProductReference.objects.filter( + location_id__in=all_location_ids, + product=self._product, + ).values_list("location_id", "status"), + ) + for location in saved: + if location.location_id not in existing_product_refs: + assoc = location.get_association_data() + status = ( + ProductLocationStatus.Mitigated + if location.location_id in new_ref_statuses and location.location_id not in newly_active + else ProductLocationStatus.Active + ) + all_product_refs.append(LocationProductReference( + location_id=location.location_id, + product=self._product, + status=status, + relationship=assoc.relationship_type, + relationship_data=assoc.relationship_data, + )) + existing_product_refs[location.location_id] = status + # A new Active finding ref makes its location Active on the product again. + product_refs_to_reactivate = { + location_id for location_id in newly_active + if existing_product_refs.get(location_id) == ProductLocationStatus.Mitigated + } # Bulk create references - all_finding_refs = self._drop_finding_refs_for_missing_findings(all_finding_refs) if all_finding_refs: LocationFindingReference.objects.bulk_create( all_finding_refs, batch_size=1000, ignore_conflicts=True, @@ -252,6 +285,12 @@ def _persist_locations(self) -> None: LocationProductReference.objects.bulk_create( all_product_refs, batch_size=1000, ignore_conflicts=True, ) + if product_refs_to_reactivate: + LocationProductReference.objects.filter( + location_id__in=product_refs_to_reactivate, + product=self._product, + status=ProductLocationStatus.Mitigated, + ).update(status=ProductLocationStatus.Active) # Trigger bulk tag inheritance only when the Location's product # membership actually changed. New product refs are the only thing diff --git a/unittests/test_location_reference_status_on_create.py b/unittests/test_location_reference_status_on_create.py new file mode 100644 index 00000000000..ac2086098ee --- /dev/null +++ b/unittests/test_location_reference_status_on_create.py @@ -0,0 +1,104 @@ +""" +A location reference the importer creates carries the status of what it references. + +# Regression: an import created every new finding reference as Active, even for a finding that arrived +# already mitigated, and never reactivated an Asset reference that was Mitigated when a new Active finding +# reference joined it. The Locations list then counted closed findings as active on a location whose Asset +# reference read Mitigated. + +# Only a finding the report marks mitigated gets a Mitigated reference. An inactive, unverified, duplicate, +# False Positive, Out Of Scope or Risk Accepted finding keeps the Active reference imports have always created +# (test_import_reimport pins that contract for inactive imports). +""" + +from django.test import override_settings +from django.utils import timezone + +from dojo.importers.location_manager import LocationManager +from dojo.location.models import LocationFindingReference, LocationProductReference +from dojo.location.status import FindingLocationStatus, ProductLocationStatus +from dojo.models import Dojo_User, Engagement, Finding, Product, Product_Type, Test, Test_Type +from dojo.tools.locations import LocationData +from dojo.url.models import URL + +from .dojo_test_case import DojoTestCase + +LOCATION = "https://status-on-create.example.com/login" + + +@override_settings(V3_FEATURE_LOCATIONS=True) +class TestLocationReferenceStatusOnCreate(DojoTestCase): + + def setUp(self): + super().setUp() + now = timezone.now() + test_type = Test_Type.objects.get_or_create(name="Manual Test")[0] + product_type = Product_Type.objects.create(name="Status on create org") + self.product = Product.objects.create(name="Status on create asset", description="test", prod_type=product_type) + engagement = Engagement.objects.create(name="Status on create", product=self.product, target_start=now, target_end=now) + self.test = Test.objects.create(engagement=engagement, test_type=test_type, target_start=now, target_end=now) + self.reporter = Dojo_User.objects.create(username="status-on-create-reporter") + + def _finding(self, title, **status): + return Finding.objects.create(test=self.test, title=title, severity="High", reporter=self.reporter, **status) + + def _import(self, *findings): + """Persist ``LOCATION`` on each finding the way an import does.""" + manager = LocationManager(self.product) + manager.set_test(self.test) + for finding in findings: + manager.record_locations_for_finding(finding, [LocationData.url(url=LOCATION)]) + manager.persist() + + def _finding_ref_status(self, finding): + return LocationFindingReference.objects.get(finding=finding).status + + def _asset_ref_status(self): + return LocationProductReference.objects.get(product=self.product).status + + def test_a_new_finding_reference_is_mitigated_only_for_a_mitigated_finding(self): + cases = [ + ("open finding", {"active": True}, FindingLocationStatus.Active), + ("mitigated finding", {"active": False, "is_mitigated": True, "mitigated": timezone.now()}, FindingLocationStatus.Mitigated), + ("inactive, not mitigated finding", {"active": False, "verified": False}, FindingLocationStatus.Active), + ("duplicate", {"active": False, "duplicate": True}, FindingLocationStatus.Active), + ("false positive", {"active": False, "false_p": True}, FindingLocationStatus.Active), + ("out of scope", {"active": False, "out_of_scope": True}, FindingLocationStatus.Active), + ("risk accepted", {"active": False, "risk_accepted": True}, FindingLocationStatus.Active), + ] + for label, status, expected in cases: + with self.subTest(label): + finding = self._finding(label, **status) + self._import(finding) + persisted = self._finding_ref_status(finding) + self.assertEqual(expected, persisted, f"{label}: expected {expected}, persisted {persisted}") + + def test_a_new_asset_reference_is_active_only_when_a_finding_reference_is_active(self): + cases = [ + ("only a mitigated finding", [{"active": False, "is_mitigated": True}], ProductLocationStatus.Mitigated), + ("only an inactive, not mitigated finding", [{"active": False}], ProductLocationStatus.Active), + ("an open finding", [{"active": True}], ProductLocationStatus.Active), + ("an open and a mitigated finding", [{"active": False, "is_mitigated": True}, {"active": True}], ProductLocationStatus.Active), + ] + for label, statuses, expected in cases: + with self.subTest(label): + LocationProductReference.objects.filter(product=self.product).delete() + findings = [self._finding(f"{label} {index}", **status) for index, status in enumerate(statuses)] + self._import(*findings) + persisted = self._asset_ref_status() + self.assertEqual(expected, persisted, f"{label}: expected {expected}, persisted {persisted}") + + def test_a_mitigated_asset_reference_is_reactivated_by_a_new_active_finding_reference(self): + cases = [ + ("open finding", {"active": True}, ProductLocationStatus.Active), + ("mitigated finding", {"active": False, "is_mitigated": True}, ProductLocationStatus.Mitigated), + ] + location = URL.create_location_from_value(LOCATION).location + for label, status, expected in cases: + with self.subTest(label): + LocationProductReference.objects.update_or_create( + location=location, product=self.product, defaults={"status": ProductLocationStatus.Mitigated}, + ) + self._import(self._finding(label, **status)) + persisted = self._asset_ref_status() + self.assertEqual(expected, persisted, f"{label}: expected {expected}, persisted {persisted}") From cea9464db0429cb2fec1529a469eb4f43043e9a2 Mon Sep 17 00:00:00 2001 From: Cody Maffucci <46459665+Maffooch@users.noreply.github.com> Date: Thu, 12 Nov 2026 21:49:40 +0000 Subject: [PATCH 20/61] fix(celery): give the outer task its cache back after a nested foreground task (#16821) * fix(celery): give the outer task its cache back after a nested foreground task A task run in the foreground inside another one (dojo_dispatch_task with force_sync=True) went through DojoAsyncTask.__call__, which installed a fresh task cache and then set the thread's cache to None when it ended. The outer task ran on with no task cache, so every memoized read after that point went back to the database or the shared cache. begin_task_cache() now returns the cache it replaced and end_task_cache() restores it. A top-level task still starts with a fresh cache and leaves None behind; a nested one still gets a cache of its own. Co-Authored-By: Claude Opus 5.5 * docs(celery): say what a nested task does to invalidation; make the nested-task test rerunnable begin_task_cache and end_task_cache now say that an outer task keeps what it memoized for the whole task, and that an invalidation inside a nested foreground task clears only the nested task's cache. The test registered its two tasks on the global Celery app inside the test body. Celery hands back an already registered task for a name, so a second run in one process called the first run's closures. The test now drops both names before registering them and again when it ends. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- dojo/celery.py | 7 +++-- dojo/request_cache/__init__.py | 17 +++++++++-- unittests/test_caching.py | 55 +++++++++++++++++++++++++++++++++- 3 files changed, 72 insertions(+), 7 deletions(-) diff --git a/dojo/celery.py b/dojo/celery.py index 2338e7e9f17..0974ba6c3b3 100644 --- a/dojo/celery.py +++ b/dojo/celery.py @@ -55,8 +55,9 @@ def __call__(self, *args, **kwargs): return None # Install a fresh task-scoped request-cache (begin) and drop it afterwards # (end), so cache_for_request_or_task can memoize within this task without a - # value leaking to the next task on this reused worker thread. - begin_task_cache() + # value leaking to the next task on this reused worker thread. A task run in the + # foreground inside another one gives the outer task its cache back when it ends. + outer_task_cache = begin_task_cache() try: if "async_user_id" not in kwargs: return super().__call__(*args, **kwargs) @@ -70,7 +71,7 @@ def __call__(self, *args, **kwargs): with crum.impersonate(user): return super().__call__(*args, **kwargs) finally: - end_task_cache() + end_task_cache(outer_task_cache) def apply_async(self, args=None, kwargs=None, **options): """Override apply_async to inject user context and track tasks.""" diff --git a/dojo/request_cache/__init__.py b/dojo/request_cache/__init__.py index 20606136920..ae2d01baac6 100644 --- a/dojo/request_cache/__init__.py +++ b/dojo/request_cache/__init__.py @@ -41,16 +41,27 @@ def begin_task_cache(): # noqa: RUF067 """ Install a fresh task cache. Called at the START of every task so a task never inherits a cache populated by a prior task on the same (reused) worker thread. + + Returns the cache it replaced (``None`` at a top-level task boundary). A task run + in the foreground inside another one (``dojo_dispatch_task(..., force_sync=True)``) + passes it back to ``end_task_cache``, so the outer task keeps its cache: what the + outer task memoized stays memoized for the whole task, and an invalidation made + inside the nested task clears only the nested task's cache. """ + previous = get_task_cache() _task_cache_local.cache = _TaskCache() + return previous -def end_task_cache(): # noqa: RUF067 +def end_task_cache(previous=None): # noqa: RUF067 """ Drop the task cache. Called at the END of every task so nothing lingers on the - thread after the task completes. + thread after the task completes. ``previous`` is what ``begin_task_cache`` + returned: the enclosing task's cache when this task ran nested in it, else ``None``. + A nested task's cache, and anything it invalidated, is dropped here; the enclosing + task gets back the cache it had, with what it had memoized. """ - _task_cache_local.cache = None + _task_cache_local.cache = previous cache_args_kwargs_marker = object() # noqa: RUF067 marker for separating args from kwargs (needs to be global) diff --git a/unittests/test_caching.py b/unittests/test_caching.py index a1e8c1487c1..67a1e13bc2e 100644 --- a/unittests/test_caching.py +++ b/unittests/test_caching.py @@ -10,8 +10,10 @@ invalidate_dojo_settings_cache, model_to_cache_dict, ) +from dojo.celery import DojoAsyncTask, app +from dojo.celery_dispatch import dojo_dispatch_task from dojo.models import Dojo_User, System_Settings -from dojo.request_cache import begin_task_cache, cache_for_request_or_task, end_task_cache +from dojo.request_cache import begin_task_cache, cache_for_request_or_task, end_task_cache, get_task_cache from .dojo_test_case import DojoTestCase @@ -179,3 +181,54 @@ def test_end_task_cache_clears(self): with impersonate(self.user_a): self.getter() # no task cache and no request -> recomputed self.assertEqual(self.calls["n"], 2) + + +class NestedTaskCacheTest(DojoTestCase): + + """ + A task run in the foreground inside another task (``dojo_dispatch_task(..., force_sync=True)``) + gets a task cache of its own and gives the outer task's cache back when it ends. + """ + + # Regression: the nested task's end set the thread's task cache to None, so every memoized read + # for the rest of the outer task (a chunked connector reimport, after its first batch's + # synchronous prioritization) went back to the database or the shared cache. + def test_nested_foreground_task_keeps_the_outer_task_cache(self): + seen = {} + calls = {"n": 0} + + @cache_for_request_or_task + def getter(): + calls["n"] += 1 + return calls["n"] + + # Registered for this test only. Celery hands back an already registered task for a name it + # has seen, so a second run in one process would otherwise call the first run's closures. + for name in ("unittests.test_caching.nested_inner", "unittests.test_caching.nested_outer"): + app.tasks.pop(name, None) + self.addCleanup(app.tasks.pop, name, None) + + @app.task(base=DojoAsyncTask, name="unittests.test_caching.nested_inner") + def inner(*_args, **_kwargs): # absorbs the dispatch kwargs (force_sync, async_user_id) + seen["inner"] = get_task_cache() + + @app.task(base=DojoAsyncTask, name="unittests.test_caching.nested_outer") + def outer(*_args, **_kwargs): + seen["outer_before"] = get_task_cache() + getter() + dojo_dispatch_task(inner, force_sync=True) + seen["outer_after"] = get_task_cache() + getter() + + outer() + + self.assertIsNotNone(seen["outer_before"]) + self.assertIsNotNone(seen["inner"]) + self.assertIsNot(seen["inner"], seen["outer_before"], msg="the nested task must not share the outer cache") + self.assertIs( + seen["outer_after"], + seen["outer_before"], + msg=f"outer task cache after the nested task: {seen['outer_after']!r}, before: {seen['outer_before']!r}", + ) + self.assertEqual(calls["n"], 1, msg=f"memoized read computed {calls['n']} times in one task") + self.assertIsNone(get_task_cache(), msg="no task cache may outlive the outer task") From 9591527a4b55ed076be17a88e5b96dd871308a74 Mon Sep 17 00:00:00 2001 From: Cody Maffucci <46459665+Maffooch@users.noreply.github.com> Date: Thu, 12 Nov 2026 22:03:50 +0000 Subject: [PATCH 21/61] fix(jira): let a Jira project turn off SLA notification comments (#16799) * fix(jira): let a Jira project suppress SLA notification comments A Jira project's "Send SLA notifications as comment?" could only turn SLA comments on. The daily SLA job comments when the Jira instance's global setting or the project's setting is on, so a project that unchecked it still got an overdue comment on every breached issue each day while the instance setting was on. Add JIRA_Project.suppress_sla_notification_comments (default False, also as a database default). When it is set the SLA job posts no Jira comment for that project; email, in-app and other SLA notifications still go out. Otherwise the rule is unchanged: comment when the instance or the project asks for it. product_jira_sla_notification keeps its type and meaning, and migration 0309 only adds the column, so no row is rewritten and API clients that never send the new field behave as before. Both set at once is rejected by the model's clean() and the API serializer. The classic form shows one select (Inherit from Jira instance / Always send / Never send) mapped onto the two booleans. The instance setting's help text now says how a project overrides it. Co-Authored-By: Claude Opus 5.5 * docs(jira): describe the three SLA comment choices and the API fields Co-Authored-By: Claude Opus 5.5 * fix(jira): a classic form post with the old SLA comment checkbox still sends The classic Jira project form replaced the "Send SLA notifications as comment?" checkbox with a three-choice select. A post that still sends the old product_jira_sla_notification=on and no choice (a script or a saved form) was saved as Inherit and the "on" was dropped. It now maps to Always send, and counts as a change. A post with neither keeps the stored setting. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- .../connectors/downstream/PRO__jira_guide.md | 10 ++- .../connectors/os_jira/os__jira_guide.md | 10 ++- ...ject_suppress_sla_notification_comments.py | 32 +++++++++ dojo/jira/api/serializers.py | 9 +++ dojo/jira/forms.py | 68 +++++++++++++++++-- dojo/jira/models.py | 18 ++++- dojo/notifications/helper.py | 12 +++- unittests/dojo_test_case.py | 6 +- unittests/test_jira_config_engagement.py | 8 +-- unittests/test_jira_config_engagement_epic.py | 2 +- unittests/test_jira_config_product.py | 66 +++++++++++++++++- unittests/test_notifications_sla.py | 51 +++++++++++++- unittests/test_rest_framework.py | 42 ++++++++++++ unittests/test_ticket_comments.py | 2 +- 14 files changed, 311 insertions(+), 25 deletions(-) create mode 100644 dojo/db_migrations/0309_jira_project_suppress_sla_notification_comments.py diff --git a/docs/content/connectors/downstream/PRO__jira_guide.md b/docs/content/connectors/downstream/PRO__jira_guide.md index 74c20db4887..b28a6f339ac 100644 --- a/docs/content/connectors/downstream/PRO__jira_guide.md +++ b/docs/content/connectors/downstream/PRO__jira_guide.md @@ -44,7 +44,7 @@ The one exception is **Engagement epics**. The Downstream Connector has no conce * Each **Jira Project** configuration becomes an issue tracker mapping, keeping its project key and issue type, and stays assigned to the same Asset or Engagement. * **Push All Issues** is preserved: projects that had it enabled keep pushing automatically. * **Push Notes** becomes **Push Notes as Comments** on the mapping. Private notes are still never posted. -* **Send SLA Notifications As Comments** becomes **Post SLA Notifications as Comments** on the mapping. It is turned on when the Jira instance's global SLA notification setting or the project's own setting is on, as classic Jira decided it. +* **Send SLA Notifications As Comments** becomes **Post SLA Notifications as Comments** on the mapping. It is turned on as classic Jira decides it: when the project is set to **Always Send**, or when it inherits the Jira instance's global SLA notification setting and that setting is on. A project set to **Never Send** gets it turned off. * **Send Risk Acceptance Expiration Notifications As Comment** becomes **Post Risk Acceptance Changes as Comments** on the mapping. Like the classic setting, it covers every risk acceptance comment: accepted, removed, reinstated, about to expire, and expired when the risk acceptance reactivates its Findings on expiry. See [Post SLA notifications and risk acceptance changes as comments](/connectors/toolreference/jira/#post-sla-notifications-and-risk-acceptance-changes-as-comments-optional). * **Custom fields**, **close/reopen transition fields**, **component**, **default assignee**, and **labels** are converted to field mappings. Where you used *Add Vulnerability Id as a Jira label*, that becomes a label mapping too. * A **custom issue template** directory becomes a ticket template. The stock templates are not copied, because the connector already ships equivalents. @@ -308,7 +308,13 @@ If enabled, Jira comments will populate on the associated Finding in DefectDojo, #### Send SLA Notifications As Comments -If enabled, any Issue which breaches DefectDojo's Service Level Agreement rules will have comments added to the Jira issue indicating this. These comments will be posted daily until the Issue is resolved. +Decides whether Issues that approach or breach DefectDojo's Service Level Agreement rules get comments added to the Jira issue indicating this. These comments are posted daily until the Issue is resolved. The setting has three choices: + +* **Inherit From Jira Instance** (the default): follows the Jira instance's **Send Global SLA Notifications as Comment** setting. +* **Always Send**: posts the comments for this Asset, even when the Jira instance setting is off. +* **Never Send**: posts no comments for this Asset, even when the Jira instance setting is on. Email, in-app and other SLA notifications are still sent. + +Through the API (`/api/v2/jira_projects/`) the setting is two booleans. `product_jira_sla_notification: true` is **Always Send** and `suppress_sla_notification_comments: true` is **Never Send**. With both `false` the project follows the Jira instance. They cannot both be `true`. Existing projects and API clients that do not send `suppress_sla_notification_comments` keep working as before: the field defaults to `false`. Service Level Agreements can be configured under **Settings > Finding Workflow > Service Level Agreements** in DefectDojo and assigned to each Asset. diff --git a/docs/content/connectors/os_jira/os__jira_guide.md b/docs/content/connectors/os_jira/os__jira_guide.md index 5bb4667fa2a..d20538b91d6 100644 --- a/docs/content/connectors/os_jira/os__jira_guide.md +++ b/docs/content/connectors/os_jira/os__jira_guide.md @@ -124,7 +124,7 @@ If you leave this field blank, it will default to **Jira\_full.** ​ 7. Select one or more Jira Resolution types which will change the status of a Finding to False Positive (when the Resolution is triggered on the Issue). If you don't wish to use this automation, you can leave the field blank. ​ -8. Decide whether you wish to send SLA Notifications as a comment on a Jira issue. +8. Decide whether you wish to send SLA Notifications as a comment on a Jira issue. Each Asset's Jira settings can override this choice (see [Send SLA Notifications As Comments](#send-sla-notifications-as-comments)). ​ 9. Decide whether you wish to automatically sync Findings with Jira. If this is enabled, Jira Issues will automatically be kept in sync with the related Findings. If this is not enabled, you will need to manually push any changes made to a Finding after the Issue has been created in Jira. ​ @@ -261,7 +261,13 @@ If enabled, Jira comments will populate on the associated Finding in DefectDojo, #### Send SLA Notifications As Comments -If enabled, any Issue which breaches DefectDojo's Service Level Agreement rules will have comments added to the Jira issue indicating this. These comments will be posted daily until the Issue is resolved. +Decides whether Issues that approach or breach DefectDojo's Service Level Agreement rules get comments added to the Jira issue indicating this. These comments are posted daily until the Issue is resolved. The setting has three choices: + +* **Inherit from Jira instance** (the default): follows the Jira instance's **Globally send SLA notifications as comment?** setting. +* **Always send**: posts the comments for this Asset, even when the Jira instance setting is off. +* **Never send**: posts no comments for this Asset, even when the Jira instance setting is on. Email, in-app and other SLA notifications are still sent. + +Through the API (`/api/v2/jira_projects/`) the setting is two booleans. `product_jira_sla_notification: true` is **Always send** and `suppress_sla_notification_comments: true` is **Never send**. With both `false` the project follows the Jira instance. They cannot both be `true`. Existing projects and API clients that do not send `suppress_sla_notification_comments` keep working as before: the field defaults to `false`. Service Level Agreements can be configured under **Configuration \> SLA Configuration** in DefectDojo and assigned to each Asset. diff --git a/dojo/db_migrations/0309_jira_project_suppress_sla_notification_comments.py b/dojo/db_migrations/0309_jira_project_suppress_sla_notification_comments.py new file mode 100644 index 00000000000..e0c97e2b664 --- /dev/null +++ b/dojo/db_migrations/0309_jira_project_suppress_sla_notification_comments.py @@ -0,0 +1,32 @@ +# Generated by Django 5.2.17 on 2026-11-11 21:47 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + """ + Add JIRA_Project.suppress_sla_notification_comments, which turns the SLA notification comments off for one + Jira project whatever the Jira instance's global setting says. + + product_jira_sla_notification keeps its meaning and its values: no row is rewritten. The new column defaults to + False in the database too, so every existing project, and any INSERT that does not name the column, behaves as + before. The instance setting's help text now says how a project overrides it. + """ + + dependencies = [ + ("dojo", "0308_tag_name_upper_index"), + ] + + operations = [ + migrations.AddField( + model_name="jira_project", + name="suppress_sla_notification_comments", + field=models.BooleanField(blank=True, db_default=False, default=False, help_text="Overrides the Jira instance's 'Globally send SLA notifications as comment?' setting: no SLA notification is posted as a Jira comment for this Jira project. Email, in-app and other SLA notifications are still sent.", verbose_name="Never send SLA notifications as comment"), + ), + migrations.AlterField( + model_name="jira_instance", + name="global_jira_sla_notification", + field=models.BooleanField(default=True, help_text="Each Jira project configuration can override this: 'Send SLA notifications as comment?' sends them even when this is off, and 'Never send SLA notifications as comment' stops them even when this is on.", verbose_name="Globally send SLA notifications as comment?"), + ), + ] diff --git a/dojo/jira/api/serializers.py b/dojo/jira/api/serializers.py index 53f76fbe779..18c55fcbdc3 100644 --- a/dojo/jira/api/serializers.py +++ b/dojo/jira/api/serializers.py @@ -4,6 +4,7 @@ from rest_framework import serializers from dojo.jira import services as jira_services +from dojo.jira.models import sla_notification_conflict, sla_notification_conflict_message from dojo.jira.queries import get_authorized_jira_projects from dojo.models import ( JIRA_Instance, @@ -105,6 +106,14 @@ def validate(self, data): except json.JSONDecodeError as e: raise serializers.ValidationError({json_field: f"Invalid JSON: {e}"}) from e + # A field the request leaves out keeps its stored value (or its default on create). + send, suppress = ( + data.get(field, getattr(self.instance, field, False)) + for field in ("product_jira_sla_notification", "suppress_sla_notification_comments") + ) + if sla_notification_conflict(send, suppress): + raise serializers.ValidationError({"suppress_sla_notification_comments": sla_notification_conflict_message()}) + return data diff --git a/dojo/jira/forms.py b/dojo/jira/forms.py index a2b40c3de7a..7dd22b60f34 100644 --- a/dojo/jira/forms.py +++ b/dojo/jira/forms.py @@ -7,6 +7,7 @@ from django.core import validators from django.core.exceptions import ValidationError from django.urls import reverse +from django.utils.translation import gettext_lazy as _ from dojo.jira import services as jira_services from dojo.models import ( @@ -118,8 +119,45 @@ class Meta: fields = ["id"] +# The form shows the SLA comment setting as one choice, stored on two booleans: +# (product_jira_sla_notification, suppress_sla_notification_comments). Neither set follows the JIRA instance. +SLA_NOTIFICATION_COMMENT_FIELDS = { + "inherit": (False, False), + "always": (True, False), + "never": (False, True), +} + + +def sla_notification_comment_choice(jira_project): + # Suppress wins when both are set, as it does in the SLA job. + if jira_project.suppress_sla_notification_comments: + return "never" + if jira_project.product_jira_sla_notification: + return "always" + return "inherit" + + +class SlaNotificationCommentsField(forms.ChoiceField): + + """A post without the choice keeps the stored setting (see JIRAProjectForm.clean), so it is no change.""" + + def has_changed(self, initial, data): + return bool(data) and super().has_changed(initial, data) + + class JIRAProjectForm(forms.ModelForm): inherit_from_product = forms.BooleanField(label="inherit JIRA settings from product", required=False) + sla_notification_comments = SlaNotificationCommentsField( + required=False, + initial="inherit", + choices=( + ("inherit", _("Inherit from Jira instance")), + ("always", _("Always send")), + ("never", _("Never send")), + ), + label=_("Send SLA notifications as comment?"), + help_text=_("Inherit from Jira instance follows the Jira instance's 'Globally send SLA notifications as comment?' setting. Always send and Never send override it for this Jira project."), + ) jira_instance = forms.ModelChoiceField(queryset=JIRA_Instance.objects.all(), label="JIRA Instance", required=False) issue_template_dir = forms.ChoiceField(required=False, choices=JIRA_TEMPLATE_CHOICES, @@ -130,7 +168,7 @@ class JIRAProjectForm(forms.ModelForm): class Meta: model = JIRA_Project exclude = ["product", "engagement"] - fields = ["inherit_from_product", "jira_instance", "project_key", "issue_template_dir", "epic_issue_type_name", "component", "custom_fields", "close_transition_fields", "reopen_transition_fields", "jira_labels", "default_assignee", "enabled", "add_vulnerability_id_to_jira_label", "push_all_issues", "enable_engagement_epic_mapping", "push_notes", "product_jira_sla_notification", "risk_acceptance_expiration_notification"] + fields = ["inherit_from_product", "jira_instance", "project_key", "issue_template_dir", "epic_issue_type_name", "component", "custom_fields", "close_transition_fields", "reopen_transition_fields", "jira_labels", "default_assignee", "enabled", "add_vulnerability_id_to_jira_label", "push_all_issues", "enable_engagement_epic_mapping", "push_notes", "sla_notification_comments", "risk_acceptance_expiration_notification"] def __init__(self, *args, **kwargs): # if the form is shown for an engagement, we set a placeholder text around inherited settings from product @@ -139,6 +177,9 @@ def __init__(self, *args, **kwargs): self.engagement = kwargs.pop("engagement", None) super().__init__(*args, **kwargs) + if self.instance.pk: + self.initial.setdefault("sla_notification_comments", sla_notification_comment_choice(self.instance)) + self.fields["component"].help_text = ( "Comma-separate multiple components to assign more than one to the JIRA issue, " "e.g. 'Security, DevSecOps'." @@ -179,7 +220,7 @@ def __init__(self, *args, **kwargs): self.fields["push_all_issues"].disabled = False self.fields["enable_engagement_epic_mapping"].disabled = False self.fields["push_notes"].disabled = False - self.fields["product_jira_sla_notification"].disabled = False + self.fields["sla_notification_comments"].disabled = False self.fields["risk_acceptance_expiration_notification"].disabled = False elif self.product: @@ -206,7 +247,7 @@ def __init__(self, *args, **kwargs): self.initial["push_all_issues"] = jira_project_product.push_all_issues self.initial["enable_engagement_epic_mapping"] = jira_project_product.enable_engagement_epic_mapping self.initial["push_notes"] = jira_project_product.push_notes - self.initial["product_jira_sla_notification"] = jira_project_product.product_jira_sla_notification + self.initial["sla_notification_comments"] = sla_notification_comment_choice(jira_project_product) self.initial["risk_acceptance_expiration_notification"] = jira_project_product.risk_acceptance_expiration_notification self.fields["jira_instance"].disabled = True @@ -224,7 +265,7 @@ def __init__(self, *args, **kwargs): self.fields["push_all_issues"].disabled = True self.fields["enable_engagement_epic_mapping"].disabled = True self.fields["push_notes"].disabled = True - self.fields["product_jira_sla_notification"].disabled = True + self.fields["sla_notification_comments"].disabled = True self.fields["risk_acceptance_expiration_notification"].disabled = True else: @@ -237,10 +278,29 @@ def __init__(self, *args, **kwargs): self.fields["project_key"].required = True self.fields["epic_issue_type_name"].required = True + def legacy_sla_notification_posted(self): + """True when the post checks the product_jira_sla_notification checkbox the SLA comment select replaced.""" + return self.data.get(self.add_prefix("product_jira_sla_notification"), "").lower() in {"on", "true", "1"} + + def has_changed(self): + # The legacy checkbox is no form field, so it would not count as a change on its own. + return super().has_changed() or ( + self.legacy_sla_notification_posted() and sla_notification_comment_choice(self.instance) != "always" + ) + def clean(self): logger.debug("validating jira project form") cleaned_data = super().clean() + # A post without the choice keeps the stored setting, unless it sends the checkbox this select replaced + # (a script or a saved form): "on" there still means always send. The model's clean() rejects send and + # suppress together. + choice = cleaned_data.get("sla_notification_comments") + if not choice and self.legacy_sla_notification_posted(): + choice = "always" + choice = choice or sla_notification_comment_choice(self.instance) + self.instance.product_jira_sla_notification, self.instance.suppress_sla_notification_comments = SLA_NOTIFICATION_COMMENT_FIELDS[choice] + logger.debug("clean: inherit: %s", self.cleaned_data.get("inherit_from_product", False)) if not self.cleaned_data.get("inherit_from_product", False): jira_instance = self.cleaned_data.get("jira_instance") diff --git a/dojo/jira/models.py b/dojo/jira/models.py index a2d7f998edb..2f46c863396 100644 --- a/dojo/jira/models.py +++ b/dojo/jira/models.py @@ -44,7 +44,7 @@ class JIRA_Instance(models.Model): finding_text = models.TextField(null=True, blank=True, help_text=_("Additional text that will be added to the finding in Jira. For example including how the finding was created or who to contact for more information.")) accepted_mapping_resolution = models.CharField(null=True, blank=True, max_length=300, verbose_name="Risk Accepted resolution mapping", help_text=_('JIRA issues that are closed in JIRA with one of these resolutions will result in the Finding becoming Risk Accepted in Defect Dojo. JIRA issues that are closed in JIRA with one of these resolutions will result in the Finding becoming Risk Accepted in Defect Dojo. The expiration time for this Risk Acceptance will be determined by the "Risk acceptance form default days" in "System Settings". This mapping is not used when Findings are pushed to JIRA. In that case the Risk Accepted Findings are closed in JIRA and JIRA sets the default resolution.')) false_positive_mapping_resolution = models.CharField(null=True, blank=True, verbose_name="False Positive resolution mapping", max_length=300, help_text=_("JIRA issues that are closed in JIRA with one of these resolutions will result in the Finding being marked as False Positive Defect Dojo. This mapping is not used when Findings are pushed to JIRA. In that case the Finding is closed in JIRA and JIRA sets the default resolution.")) - global_jira_sla_notification = models.BooleanField(default=True, blank=False, verbose_name=_("Globally send SLA notifications as comment?"), help_text=_("This setting can be overidden at the Product level")) + global_jira_sla_notification = models.BooleanField(default=True, blank=False, verbose_name=_("Globally send SLA notifications as comment?"), help_text=_("Each Jira project configuration can override this: 'Send SLA notifications as comment?' sends them even when this is off, and 'Never send SLA notifications as comment' stops them even when this is on.")) finding_jira_sync = models.BooleanField(default=False, blank=False, verbose_name=_("Automatically sync Findings with JIRA?"), help_text=_("If enabled, this will sync changes to a Finding automatically to JIRA")) class Meta: @@ -134,6 +134,11 @@ class JIRA_Project(models.Model): epic_issue_type_name = models.CharField(max_length=64, blank=True, default="Epic", help_text=_("The name of the of structure that represents an Epic")) push_notes = models.BooleanField(default=False, blank=True) product_jira_sla_notification = models.BooleanField(default=False, blank=True, verbose_name=_("Send SLA notifications as comment?")) + # Unchecked, product_jira_sla_notification follows the JIRA instance's global_jira_sla_notification; this turns the + # comments off for this JIRA project whatever the instance says. The two cannot both be set (see clean()). + # db_default keeps an INSERT that does not name the column (a writer older than it) working. + suppress_sla_notification_comments = models.BooleanField(default=False, db_default=False, blank=True, verbose_name=_("Never send SLA notifications as comment"), + help_text=_("Overrides the Jira instance's 'Globally send SLA notifications as comment?' setting: no SLA notification is posted as a Jira comment for this Jira project. Email, in-app and other SLA notifications are still sent.")) risk_acceptance_expiration_notification = models.BooleanField(default=False, blank=True, verbose_name=_("Send Risk Acceptance expiration notifications as comment?")) enabled = models.BooleanField( verbose_name=_("Enable Connection With Jira Project"), @@ -162,6 +167,17 @@ def clean(self): if not self.jira_instance: msg = "Cannot save JIRA Project Configuration without JIRA Instance" raise ValidationError(msg) + if sla_notification_conflict(self.product_jira_sla_notification, self.suppress_sla_notification_comments): + raise ValidationError({"suppress_sla_notification_comments": sla_notification_conflict_message()}) + + +def sla_notification_conflict(send, suppress): + """A JIRA project cannot both send SLA notifications as comments and never send them.""" + return bool(send and suppress) + + +def sla_notification_conflict_message(): + return _("Choose either 'Send SLA notifications as comment?' or 'Never send SLA notifications as comment', not both.") # declare form here as we can't import forms.py due to circular imports not even locally diff --git a/dojo/notifications/helper.py b/dojo/notifications/helper.py index 81232d97f07..f1ef008b142 100644 --- a/dojo/notifications/helper.py +++ b/dojo/notifications/helper.py @@ -1662,18 +1662,24 @@ def _create_notifications(batch): jira_instance = jira_services.get_instance(finding) if jira_instance is not None: logger.debug("JIRA config for finding is %s", jira_instance) - # global config or product config set, product level takes precedence + # comment when the global config or the JIRA project config asks for it, unless the + # JIRA project suppresses SLA comments, which overrides both product_jira_sla_comment_enabled = False + product_jira_sla_comment_suppressed = False try: # TODO: see new property from #2649 to then replace, somehow not working with prefetching though. - product_jira_sla_comment_enabled = jira_services.get_project(finding).product_jira_sla_notification + jira_project = jira_services.get_project(finding) + product_jira_sla_comment_enabled = jira_project.product_jira_sla_notification + product_jira_sla_comment_suppressed = jira_project.suppress_sla_notification_comments except Exception as e: logger.error("The product is not linked to a JIRA configuration! Something is weird here.") logger.error("Error is: %s", e) jiraconfig_sla_notification_enabled = jira_instance.global_jira_sla_notification - if jiraconfig_sla_notification_enabled or product_jira_sla_comment_enabled: + if product_jira_sla_comment_suppressed: + logger.debug("SLA notifications as JIRA comments are suppressed for this JIRA project") + elif jiraconfig_sla_notification_enabled or product_jira_sla_comment_enabled: logger.debug("Global setting %s -- Product setting %s", jiraconfig_sla_notification_enabled, product_jira_sla_comment_enabled) do_jira_sla_comment = True logger.debug(f"JIRA issue is {jira_issue.jira_key}") diff --git a/unittests/dojo_test_case.py b/unittests/dojo_test_case.py index b6d6b0d864b..8eac205dfda 100644 --- a/unittests/dojo_test_case.py +++ b/unittests/dojo_test_case.py @@ -284,7 +284,7 @@ def get_new_product_with_jira_project_data(self): "jira-project-form-epic_issue_type_name": "Epic", "jira-project-form-enabled": "True", "jira-project-form-push_notes": "on", - "jira-project-form-product_jira_sla_notification": "on", + "jira-project-form-sla_notification_comments": "always", "jira-project-form-custom_fields": "null", "sla_configuration": 1, @@ -317,7 +317,7 @@ def get_product_with_jira_project_data(self, product): "jira-project-form-epic_issue_type_name": "Epic", "jira-project-form-enabled": "True", "jira-project-form-push_notes": "on", - "jira-project-form-product_jira_sla_notification": "on", + "jira-project-form-sla_notification_comments": "always", "jira-project-form-custom_fields": "null", "sla_configuration": 1, @@ -334,7 +334,7 @@ def get_product_with_jira_project_data2(self, product): "jira-project-form-epic_issue_type_name": "Epic", "jira-project-form-enabled": "True", "jira-project-form-push_notes": "on", - "jira-project-form-product_jira_sla_notification": "on", + "jira-project-form-sla_notification_comments": "always", "jira-project-form-custom_fields": "null", "sla_configuration": 1, diff --git a/unittests/test_jira_config_engagement.py b/unittests/test_jira_config_engagement.py index aaabfdddc5e..1f93d85f7ee 100644 --- a/unittests/test_jira_config_engagement.py +++ b/unittests/test_jira_config_engagement.py @@ -30,7 +30,7 @@ def get_new_engagement_with_jira_project_data(self): "jira-project-form-project_key": "IUNSEC", "jira-project-form-epic_issue_type_name": "Epic", "jira-project-form-enabled": "True", - "jira-project-form-product_jira_sla_notification": "on", + "jira-project-form-sla_notification_comments": "always", "jira-project-form-custom_fields": "null", } @@ -48,7 +48,7 @@ def get_new_engagement_with_jira_project_data_and_epic_mapping(self): "jira-project-form-project_key": "IUNSEC", "jira-project-form-epic_issue_type_name": "Epic", "jira-project-form-enabled": "True", - "jira-project-form-product_jira_sla_notification": "on", + "jira-project-form-sla_notification_comments": "always", "jira-project-form-enable_engagement_epic_mapping": "on", "jira-epic-form-push_to_jira": "on", "jira-project-form-custom_fields": "null", @@ -88,7 +88,7 @@ def get_engagement_with_jira_project_data(self, engagement): "jira-project-form-project_key": "ISEC", "jira-project-form-epic_issue_type_name": "Epic", "jira-project-form-enabled": "True", - "jira-project-form-product_jira_sla_notification": "on", + "jira-project-form-sla_notification_comments": "always", "jira-project-form-custom_fields": "null", } @@ -106,7 +106,7 @@ def get_engagement_with_jira_project_data2(self, engagement): "jira-project-form-project_key": "ISEC2", "jira-project-form-epic_issue_type_name": "Epic", "jira-project-form-enabled": "True", - "jira-project-form-product_jira_sla_notification": "on", + "jira-project-form-sla_notification_comments": "always", "jira-project-form-custom_fields": "null", } diff --git a/unittests/test_jira_config_engagement_epic.py b/unittests/test_jira_config_engagement_epic.py index 7ba1d287062..8e5b08a3d2d 100644 --- a/unittests/test_jira_config_engagement_epic.py +++ b/unittests/test_jira_config_engagement_epic.py @@ -66,7 +66,7 @@ def get_new_engagement_with_jira_project_data_and_epic_mapping(self): "jira-project-form-project_key": "NTEST", "jira-project-form-epic_issue_type_name": "Epic", "jira-project-form-enabled": "True", - "jira-project-form-product_jira_sla_notification": "on", + "jira-project-form-sla_notification_comments": "always", "jira-project-form-enable_engagement_epic_mapping": "on", "jira-epic-form-push_to_jira": "on", } diff --git a/unittests/test_jira_config_product.py b/unittests/test_jira_config_product.py index 52387c29d23..b22be367c25 100644 --- a/unittests/test_jira_config_product.py +++ b/unittests/test_jira_config_product.py @@ -3,12 +3,14 @@ from unittest.mock import call, patch import requests +from django.core.exceptions import ValidationError from django.urls import reverse from django.utils.http import urlencode from jira.exceptions import JIRAError +from parameterized import parameterized import dojo.jira.helper as jira_helper -from dojo.models import JIRA_Instance, Product +from dojo.models import JIRA_Instance, JIRA_Project, Product from unittests.dojo_test_case import DojoTestCase, versioned_fixtures logger = logging.getLogger(__name__) @@ -166,6 +168,68 @@ def test_add_product_with_jira_project(self, jira_mock): self.assertIsNotNone(product) self.assertEqual(jira_mock.call_count, 1) + # The classic form shows the SLA comment setting as one select with three choices, stored on two booleans: + # product_jira_sla_notification (send) and suppress_sla_notification_comments (never send). + @parameterized.expand([ + ("inherit", False, False), + ("always", True, False), + ("never", False, True), + ]) + @patch("dojo.jira.views.jira_helper.is_jira_project_valid") + def test_add_product_stores_the_sla_comment_setting(self, choice, send, suppress, jira_mock): + jira_mock.return_value = True + data = self.get_new_product_with_jira_project_data() + data["jira-project-form-sla_notification_comments"] = choice + product = self.add_product_jira_with_data(data, expected_delta_jira_project_db=1) + + persisted = JIRA_Project.objects.get(product=product) + self.assertEqual( + (persisted.product_jira_sla_notification, persisted.suppress_sla_notification_comments), (send, suppress), + msg=f"chose {choice!r}", + ) + + # The edit page shows the stored choice, so saving it again does not change it. + response = self.client.get(reverse("edit_product", args=(product.id,))) + self.assertContains(response, f'