From 1bce34223b19e3dd31e634dc4927d13e8c61b035 Mon Sep 17 00:00:00 2001 From: Artur Shiriev Date: Sun, 15 Nov 2026 13:20:04 +0300 Subject: [PATCH] fix: support Sentry redaction for independently rendered structlog records (#450) --- ...structlog-sentry-seam-stays-a-heuristic.md | 6 ++ docs/migration/to-2.x.md | 20 +++++ lite_bootstrap/instruments/logging_factory.py | 2 +- .../instruments/logging_instrument.py | 17 +++- tests/instruments/test_sentry_instrument.py | 83 ++++++++++++++++++- 5 files changed, 123 insertions(+), 5 deletions(-) diff --git a/docs/adr/0006-structlog-sentry-seam-stays-a-heuristic.md b/docs/adr/0006-structlog-sentry-seam-stays-a-heuristic.md index b089e42..b1b2d1d 100644 --- a/docs/adr/0006-structlog-sentry-seam-stays-a-heuristic.md +++ b/docs/adr/0006-structlog-sentry-seam-stays-a-heuristic.md @@ -9,6 +9,12 @@ recognises it by `startswith("{")`. An explicit marker key would pollute user-vi every logging user to serve one consumer, and a neutral third module both instruments import would abstract a sharing that does not exist. +For message-less records, JSON content alone cannot establish provenance. The +logging instrument now uses `SentryJSONRenderer`, which returns a private `str` +subclass without changing the JSON. Independent structlog configurations can opt +into that renderer too. Sentry recognizes the marked `LogRecord.msg` even when +`event` is absent or non-string; unmarked JSON in those cases stays unchanged. + The trap worth knowing before "fixing" it: structlog's own idiomatic `ProcessorFormatter.wrap_for_formatter` is incompatible with this seam. It defers rendering to handler-flush time, so `record.msg` is an `EventDict`, `sentry_sdk` builds `logentry.formatted` from diff --git a/docs/migration/to-2.x.md b/docs/migration/to-2.x.md index 9dd367b..eae923a 100644 --- a/docs/migration/to-2.x.md +++ b/docs/migration/to-2.x.md @@ -197,6 +197,26 @@ before `before_send` or `before_breadcrumb` runs. This ordering also applies to `sentry_additional_params["before_send"]`, which previously replaced built-in processing. Update callbacks that relied on the original payload ([#408](https://github.com/modern-python/lite-bootstrap/pull/408)). + +If you configure structlog yourself, use `SentryJSONRenderer` as the final processor +before a stdlib logger to redact records with a missing or non-string message: + +```python +import structlog +from lite_bootstrap.instruments.logging_instrument import SentryJSONRenderer + +structlog.configure( + processors=[SentryJSONRenderer()], + logger_factory=structlog.stdlib.LoggerFactory(), + wrapper_class=structlog.stdlib.BoundLogger, +) +``` + +It preserves JSON output and marks the string in memory. Lite-bootstrap's logging +instrument already uses it. Unmarked JSON without a string `event` stays unchanged; +its contents alone cannot distinguish structlog from unrelated stdlib JSON +([#450](https://github.com/modern-python/lite-bootstrap/issues/450)). + FastStream isolates message scopes. Set tags and breadcrumbs per message ([#409](https://github.com/modern-python/lite-bootstrap/pull/409)). diff --git a/lite_bootstrap/instruments/logging_factory.py b/lite_bootstrap/instruments/logging_factory.py index 575b9ed..f3e66ed 100644 --- a/lite_bootstrap/instruments/logging_factory.py +++ b/lite_bootstrap/instruments/logging_factory.py @@ -10,7 +10,7 @@ class _RenderedLine(str): - """A line already rendered by lite-bootstrap's structlog chain.""" + """A rendered JSON line marked for Sentry normalization before stdlib logging.""" __slots__ = () diff --git a/lite_bootstrap/instruments/logging_instrument.py b/lite_bootstrap/instruments/logging_instrument.py index 2469df5..9d1a5da 100644 --- a/lite_bootstrap/instruments/logging_instrument.py +++ b/lite_bootstrap/instruments/logging_instrument.py @@ -16,6 +16,16 @@ if import_checker.is_structlog_installed: import structlog + class SentryJSONRenderer(structlog.processors.JSONRenderer): + """Render JSON with provenance for Sentry redaction, including message-less records. + + Use as the final processor before a stdlib logger, with a serializer that + returns str. Accepts JSONRenderer's options; the marker adds no JSON fields. + """ + + def __call__(self, logger: "WrappedLogger", name: str, event_dict: "EventDict") -> str: + return _RenderedLine(super().__call__(logger, name, event_dict)) + class _RenderedLinePassthroughFormatter(structlog.stdlib.ProcessorFormatter): """Render stdlib records through the foreign chain, and emit structlog's rendered lines verbatim.""" @@ -54,6 +64,7 @@ def tracer_injection(_: "WrappedLogger", __: str, event_dict: "EventDict") -> "E __all__ = [ "LoggingConfig", "LoggingInstrument", + "SentryJSONRenderer", "tracer_injection", ] @@ -137,15 +148,15 @@ def structlog_processors(self) -> list[typing.Any]: *self.structlog_pre_chain_processors, *self.bootstrap_config.logging_extra_processors, ] - render_json = structlog.processors.JSONRenderer(serializer=_serialize_log_to_string) + render_json = SentryJSONRenderer(serializer=_serialize_log_to_string) def render_for_stdlib( logger: "WrappedLogger", method: str, event_dict: "EventDict" - ) -> tuple[tuple[_RenderedLine], dict[str, typing.Any]]: + ) -> tuple[tuple[str], dict[str, typing.Any]]: exc_info = event_dict.get("exc_info") for processor in processors: event_dict = processor(logger, method, event_dict) - line = _RenderedLine(render_json(logger, method, event_dict)) + line = render_json(logger, method, event_dict) return (line,), {"exc_info": exc_info} return [ diff --git a/tests/instruments/test_sentry_instrument.py b/tests/instruments/test_sentry_instrument.py index 77fb637..b1cce58 100644 --- a/tests/instruments/test_sentry_instrument.py +++ b/tests/instruments/test_sentry_instrument.py @@ -2,6 +2,7 @@ import dataclasses import json import logging +import sys import typing import uuid from unittest.mock import patch @@ -11,7 +12,7 @@ import structlog from sentry_sdk.integrations.logging import LoggingIntegration -from lite_bootstrap.instruments.logging_instrument import LoggingConfig, LoggingInstrument +from lite_bootstrap.instruments.logging_instrument import LoggingConfig, LoggingInstrument, SentryJSONRenderer from lite_bootstrap.instruments.sentry_instrument import enrich_sentry_event_from_structlog_log from tests.conftest import LoggingMock, SentryTestTransport @@ -703,6 +704,86 @@ def test_message_less_structlog_error_is_scrubbed_in_sentry_envelopes( sentry.teardown() +@pytest.mark.parametrize("message", [None, 42]) +def test_independent_structlog_renderer_scrubs_only_its_own_records( + minimal_sentry_config: SentryConfig, + sentry_mock: SentryTestTransport, + capsys: pytest.CaptureFixture[str], + message: int | None, +) -> None: + marker = uuid.uuid4().hex + observed: list[sentry_types.Event] = [] + observed_breadcrumbs: list[sentry_types.Breadcrumb] = [] + + def before_breadcrumb( + breadcrumb: "sentry_types.Breadcrumb", _: "sentry_types.BreadcrumbHint" + ) -> "sentry_types.Breadcrumb": + observed_breadcrumbs.append(copy.deepcopy(breadcrumb)) + return breadcrumb + + def before_send(event: "sentry_types.Event", _: "sentry_types.Hint") -> "sentry_types.Event": + observed.append(copy.deepcopy(event)) + return event + + sentry = SentryInstrument( + bootstrap_config=dataclasses.replace( + minimal_sentry_config, + sentry_before_send=before_send, + sentry_additional_params={ + **minimal_sentry_config.sentry_additional_params, + "before_breadcrumb": before_breadcrumb, + }, + ) + ) + target = logging.getLogger("independent-structlog") + handler = logging.StreamHandler(sys.stdout) + external = structlog.wrap_logger( + target, + processors=[SentryJSONRenderer()], + wrapper_class=structlog.stdlib.BoundLogger, + ) + target.addHandler(handler) + sentry.bootstrap() + try: + external.error(message, password=marker, component="worker") + external.error(password=marker, skip_sentry=True) + external.error("later error") + + first_line, skipped_line, _ = capsys.readouterr().out.splitlines() + expected_line: dict[str, typing.Any] = {"password": marker, "component": "worker"} + if message is not None: + expected_line["event"] = message + assert json.loads(first_line) == expected_line + assert json.loads(skipped_line)["skip_sentry"] is True + first, second = sentry_mock.mock_envelopes + assert marker.encode() not in first.serialize() + assert marker.encode() not in second.serialize() + assert all(marker not in json.dumps(event) for event in observed) + assert all(marker not in json.dumps(crumb, default=str) for crumb in observed_breadcrumbs) + assert all("skip_sentry" not in json.dumps(crumb, default=str) for crumb in observed_breadcrumbs) + assert observed[0]["logentry"]["formatted"] == "[No message]" + assert observed[0]["contexts"]["structlog"]["component"] == "worker" + breadcrumbs = observed[1]["breadcrumbs"] + assert isinstance(breadcrumbs, dict) + assert any(crumb.get("data", {}).get("component") == "worker" for crumb in breadcrumbs["values"]) + + target.error(first_line) + stdlib_event = sentry_mock.mock_envelopes[-1].get_event() + assert stdlib_event is not None + assert stdlib_event["logentry"]["formatted"] == first_line + assert "structlog" not in stdlib_event["contexts"] + target.error("after stdlib JSON") + later_event = sentry_mock.mock_envelopes[-1].get_event() + assert later_event is not None + later_breadcrumbs = later_event["breadcrumbs"] + assert isinstance(later_breadcrumbs, dict) + assert any(crumb.get("message") == first_line for crumb in later_breadcrumbs["values"]) + finally: + target.removeHandler(handler) + handler.close() + sentry.teardown() + + @pytest.mark.parametrize("template", ["plain %s", "{invalid %s", '{"unrelated":"%s"}']) def test_unstructured_log_keeps_sdk_truncation( minimal_sentry_config: SentryConfig, sentry_mock: SentryTestTransport, template: str