Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions docs/adr/0006-structlog-sentry-seam-stays-a-heuristic.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,12 @@ recognises it by `startswith("{")`. An explicit marker key would pollute user-vi
every logging user to serve one consumer, and a neutral third module both instruments import would
abstract a sharing that does not exist.

For message-less records, JSON content alone cannot establish provenance. The
logging instrument now uses `SentryJSONRenderer`, which returns a private `str`
subclass without changing the JSON. Independent structlog configurations can opt
into that renderer too. Sentry recognizes the marked `LogRecord.msg` even when
`event` is absent or non-string; unmarked JSON in those cases stays unchanged.

The trap worth knowing before "fixing" it: structlog's own idiomatic
`ProcessorFormatter.wrap_for_formatter` is incompatible with this seam. It defers rendering to
handler-flush time, so `record.msg` is an `EventDict`, `sentry_sdk` builds `logentry.formatted` from
Expand Down
20 changes: 20 additions & 0 deletions docs/migration/to-2.x.md
Original file line number Diff line number Diff line change
Expand Up @@ -197,6 +197,26 @@ before `before_send` or `before_breadcrumb` runs. This ordering also applies to
`sentry_additional_params["before_send"]`, which previously replaced built-in
processing. Update callbacks that relied on the original payload
([#408](https://github.com/modern-python/lite-bootstrap/pull/408)).

If you configure structlog yourself, use `SentryJSONRenderer` as the final processor
before a stdlib logger to redact records with a missing or non-string message:

```python
import structlog
from lite_bootstrap.instruments.logging_instrument import SentryJSONRenderer

structlog.configure(
processors=[SentryJSONRenderer()],
logger_factory=structlog.stdlib.LoggerFactory(),
wrapper_class=structlog.stdlib.BoundLogger,
)
```

It preserves JSON output and marks the string in memory. Lite-bootstrap's logging
instrument already uses it. Unmarked JSON without a string `event` stays unchanged;
its contents alone cannot distinguish structlog from unrelated stdlib JSON
([#450](https://github.com/modern-python/lite-bootstrap/issues/450)).

FastStream isolates message scopes. Set tags and breadcrumbs per message
([#409](https://github.com/modern-python/lite-bootstrap/pull/409)).

Expand Down
2 changes: 1 addition & 1 deletion lite_bootstrap/instruments/logging_factory.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@


class _RenderedLine(str):
"""A line already rendered by lite-bootstrap's structlog chain."""
"""A rendered JSON line marked for Sentry normalization before stdlib logging."""

__slots__ = ()

Expand Down
17 changes: 14 additions & 3 deletions lite_bootstrap/instruments/logging_instrument.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,16 @@
if import_checker.is_structlog_installed:
import structlog

class SentryJSONRenderer(structlog.processors.JSONRenderer):
"""Render JSON with provenance for Sentry redaction, including message-less records.

Use as the final processor before a stdlib logger, with a serializer that
returns str. Accepts JSONRenderer's options; the marker adds no JSON fields.
"""

def __call__(self, logger: "WrappedLogger", name: str, event_dict: "EventDict") -> str:
return _RenderedLine(super().__call__(logger, name, event_dict))

class _RenderedLinePassthroughFormatter(structlog.stdlib.ProcessorFormatter):
"""Render stdlib records through the foreign chain, and emit structlog's rendered lines verbatim."""

Expand Down Expand Up @@ -54,6 +64,7 @@ def tracer_injection(_: "WrappedLogger", __: str, event_dict: "EventDict") -> "E
__all__ = [
"LoggingConfig",
"LoggingInstrument",
"SentryJSONRenderer",
"tracer_injection",
]

Expand Down Expand Up @@ -137,15 +148,15 @@ def structlog_processors(self) -> list[typing.Any]:
*self.structlog_pre_chain_processors,
*self.bootstrap_config.logging_extra_processors,
]
render_json = structlog.processors.JSONRenderer(serializer=_serialize_log_to_string)
render_json = SentryJSONRenderer(serializer=_serialize_log_to_string)

def render_for_stdlib(
logger: "WrappedLogger", method: str, event_dict: "EventDict"
) -> tuple[tuple[_RenderedLine], dict[str, typing.Any]]:
) -> tuple[tuple[str], dict[str, typing.Any]]:
exc_info = event_dict.get("exc_info")
for processor in processors:
event_dict = processor(logger, method, event_dict)
line = _RenderedLine(render_json(logger, method, event_dict))
line = render_json(logger, method, event_dict)
return (line,), {"exc_info": exc_info}

return [
Expand Down
83 changes: 82 additions & 1 deletion tests/instruments/test_sentry_instrument.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
import dataclasses
import json
import logging
import sys
import typing
import uuid
from unittest.mock import patch
Expand All @@ -11,7 +12,7 @@
import structlog
from sentry_sdk.integrations.logging import LoggingIntegration

from lite_bootstrap.instruments.logging_instrument import LoggingConfig, LoggingInstrument
from lite_bootstrap.instruments.logging_instrument import LoggingConfig, LoggingInstrument, SentryJSONRenderer
from lite_bootstrap.instruments.sentry_instrument import enrich_sentry_event_from_structlog_log
from tests.conftest import LoggingMock, SentryTestTransport

Expand Down Expand Up @@ -703,6 +704,86 @@ def test_message_less_structlog_error_is_scrubbed_in_sentry_envelopes(
sentry.teardown()


@pytest.mark.parametrize("message", [None, 42])
def test_independent_structlog_renderer_scrubs_only_its_own_records(
minimal_sentry_config: SentryConfig,
sentry_mock: SentryTestTransport,
capsys: pytest.CaptureFixture[str],
message: int | None,
) -> None:
marker = uuid.uuid4().hex
observed: list[sentry_types.Event] = []
observed_breadcrumbs: list[sentry_types.Breadcrumb] = []

def before_breadcrumb(
breadcrumb: "sentry_types.Breadcrumb", _: "sentry_types.BreadcrumbHint"
) -> "sentry_types.Breadcrumb":
observed_breadcrumbs.append(copy.deepcopy(breadcrumb))
return breadcrumb

def before_send(event: "sentry_types.Event", _: "sentry_types.Hint") -> "sentry_types.Event":
observed.append(copy.deepcopy(event))
return event

sentry = SentryInstrument(
bootstrap_config=dataclasses.replace(
minimal_sentry_config,
sentry_before_send=before_send,
sentry_additional_params={
**minimal_sentry_config.sentry_additional_params,
"before_breadcrumb": before_breadcrumb,
},
)
)
target = logging.getLogger("independent-structlog")
handler = logging.StreamHandler(sys.stdout)
external = structlog.wrap_logger(
target,
processors=[SentryJSONRenderer()],
wrapper_class=structlog.stdlib.BoundLogger,
)
target.addHandler(handler)
sentry.bootstrap()
try:
external.error(message, password=marker, component="worker")
external.error(password=marker, skip_sentry=True)
external.error("later error")

first_line, skipped_line, _ = capsys.readouterr().out.splitlines()
expected_line: dict[str, typing.Any] = {"password": marker, "component": "worker"}
if message is not None:
expected_line["event"] = message
assert json.loads(first_line) == expected_line
assert json.loads(skipped_line)["skip_sentry"] is True
first, second = sentry_mock.mock_envelopes
assert marker.encode() not in first.serialize()
assert marker.encode() not in second.serialize()
assert all(marker not in json.dumps(event) for event in observed)
assert all(marker not in json.dumps(crumb, default=str) for crumb in observed_breadcrumbs)
assert all("skip_sentry" not in json.dumps(crumb, default=str) for crumb in observed_breadcrumbs)
assert observed[0]["logentry"]["formatted"] == "[No message]"
assert observed[0]["contexts"]["structlog"]["component"] == "worker"
breadcrumbs = observed[1]["breadcrumbs"]
assert isinstance(breadcrumbs, dict)
assert any(crumb.get("data", {}).get("component") == "worker" for crumb in breadcrumbs["values"])

target.error(first_line)
stdlib_event = sentry_mock.mock_envelopes[-1].get_event()
assert stdlib_event is not None
assert stdlib_event["logentry"]["formatted"] == first_line
assert "structlog" not in stdlib_event["contexts"]
target.error("after stdlib JSON")
later_event = sentry_mock.mock_envelopes[-1].get_event()
assert later_event is not None
later_breadcrumbs = later_event["breadcrumbs"]
assert isinstance(later_breadcrumbs, dict)
assert any(crumb.get("message") == first_line for crumb in later_breadcrumbs["values"])
finally:
target.removeHandler(handler)
handler.close()
sentry.teardown()


@pytest.mark.parametrize("template", ["plain %s", "{invalid %s", '{"unrelated":"%s"}'])
def test_unstructured_log_keeps_sdk_truncation(
minimal_sentry_config: SentryConfig, sentry_mock: SentryTestTransport, template: str
Expand Down
Loading