diff --git a/MANIFEST.in b/MANIFEST.in index 2bc96581..978126ba 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -6,5 +6,6 @@ recursive-include tests *.py *.ini *.ipynb *.md *.txt recursive-include pyegeria/view/schemas *.json recursive-include pyegeria/view/seed *.json + recursive-include pyegeria/view/static *.js *.json recursive-include sample-data/templates/basic *.md recursive-include sample-data/templates/advanced *.md diff --git a/commands/cat/load_report_specs.py b/commands/cat/load_report_specs.py index d50c2e0c..96974feb 100644 --- a/commands/cat/load_report_specs.py +++ b/commands/cat/load_report_specs.py @@ -3,11 +3,10 @@ SPDX-License-Identifier: Apache-2.0 Copyright Contributors to the ODPi Egeria project. -Refresh the in-process report-spec registry from Egeria. - -Reads ReportType collections and their QuestionSpec folders from Egeria and -merges question_specs into the runtime registry so that find_report_specs_by_* -and the MCP find_report_specs tool return Egeria-aware results. +DEPRECATED command name, kept until pyegeria 7: it now runs the spec cache's sync +(pyegeria.view.egeria_spec_cache.sync_report_specs, the same as `pyegeria report-specs sync`), which +loads the report specs held in Egeria into the cache the registry's EGERIA tier reads. It used to run +the legacy ReportType:: loader (load_egeria_report_specs), deprecated and removed in pyegeria 7. Usage (standalone): load_report_specs [--force] [--ttl ] @@ -28,12 +27,6 @@ from pyegeria.core.config import settings from pyegeria.egeria_tech_client import EgeriaTech -from pyegeria.view.base_report_formats import ( - load_egeria_report_specs, - report_spec_list, - _EGERIA_SPECS_LOADED_AT, - _EGERIA_SPECS_CACHE_TTL, -) EGERIA_USER = os.environ.get("EGERIA_USER", "erinoverview") EGERIA_USER_PASSWORD = os.environ.get("EGERIA_USER_PASSWORD", "secret") @@ -52,60 +45,76 @@ def _run( jupyter: bool = settings.Environment.egeria_jupyter, ) -> None: console = Console(width=width, force_terminal=not jupyter) - - from pyegeria.view import base_report_formats as _brf - last = getattr(_brf, "_EGERIA_SPECS_LOADED_AT", None) - effective_ttl = ttl_seconds if ttl_seconds is not None else _EGERIA_SPECS_CACHE_TTL - - if not force and last is not None: - from datetime import datetime, timezone - age = (datetime.now(timezone.utc) - last).total_seconds() - if age < effective_ttl: - console.print( - f"[green]Registry is fresh[/green] (loaded {age:.0f}s ago, TTL={effective_ttl}s). " - f"Use --force to reload anyway." - ) - return - + # The owner's direction (2026-11-14): our own callers move off the deprecated legacy ReportType:: loader + # (load_egeria_report_specs, removed in pyegeria 7). This command now runs the spec cache's sync, the + # system of record for Egeria-held report specs, exactly as `pyegeria report-specs sync` does. + console.print(DEPRECATION_WORDS) + if force or ttl_seconds is not None: + console.print("[dim]--force and --ttl applied to the legacy loader's cache and are ignored now: a sync " + "reads Egeria each time.[/dim]") + from pyegeria.view.egeria_spec_cache import SyncAlreadyRunning, sync_report_specs client = EgeriaTech(view_server, view_url, user_id=user, user_pwd=user_pass) - client.create_egeria_bearer_token(user, user_pass) - - before = set(report_spec_list()) start = time.perf_counter() - updated = load_egeria_report_specs(client, force=force, ttl_seconds=ttl_seconds) - elapsed = time.perf_counter() - start - - client.close_session() - - after = set(report_spec_list()) - new_specs = sorted(after - before) - - if updated: - console.print( - f"[green]Registry updated[/green] from Egeria in {elapsed:.2f}s. " - f"{len(new_specs)} new spec(s) added." - ) - if new_specs: - for s in new_specs: - console.print(f" + {s}") - else: - console.print( - f"[yellow]No update performed[/yellow] — Egeria returned no ReportType collections " - f"(or registry was already fresh)." - ) + try: + client.create_egeria_bearer_token(user, user_pass) + report = sync_report_specs(client) + except SyncAlreadyRunning as exc: + console.print(f"[yellow]{exc}[/yellow]") + return + finally: + try: + client.close_session() + except Exception: # noqa: BLE001 + pass + console.print(report.render()) + console.print(f"[dim]({time.perf_counter() - start:.2f}s)[/dim]") + + +#: Printed by load-report-specs / load_report_specs since it moved to the spec cache. +DEPRECATION_WORDS = ("[yellow]load-report-specs is deprecated[/yellow]: it ran the legacy ReportType:: loader, " + "which will be removed in pyegeria 7 (expected early January 2027). It now runs the spec " + "cache's sync (`pyegeria report-specs sync`), which loads Egeria-held report specs.") + + +#: An empty result from the legacy ReportType:: loader, confirmed or not (the legacy-loader ruling, 2026-11-14). +NO_LEGACY_WORDS = "No legacy ReportType collections found (Egeria-held specs load through the spec cache)." + + +def load_status_words(status: dict, elapsed: float, new_specs: list) -> str: + """Words for how the deprecated load_egeria_report_specs ended (85f5bc15 read-back R1): one sentence per + status, never "no ReportType collections" unless Egeria confirmed the end. Kept for anyone still calling the + legacy loader until pyegeria 7; this command itself now runs the spec cache's sync.""" + kind, count = status.get("status"), status.get("count") or 0 + retry_at, error = status.get("retry_at"), status.get("error") + if kind in ("loaded", "registered_unconfirmed") and not count: + # the legacy-loader ruling: never imply specs are missing (they load through the spec cache) + return NO_LEGACY_WORDS + if kind == "loaded": + return (f"[green]Registry updated[/green] from Egeria in {elapsed:.2f}s: {count} ReportType(s) read, " + f"{len(new_specs)} new spec(s) added.") + if kind == "registered_unconfirmed": + return (f"[yellow]Registered {count} spec(s)[/yellow] in {elapsed:.2f}s; Egeria didn't confirm there " + f"are no more; will re-read after {retry_at}.") + if kind == "retry_window": + return (f"[yellow]Not re-read[/yellow]: the last read wasn't confirmed complete; next read after {retry_at} " + f"(use --force).") + if kind == "fresh": + return "[green]Registry is fresh[/green]: the last confirmed load is within its TTL (use --force)." + if kind == "failed": + return f"[red]Not loaded[/red]: Egeria couldn't be queried ({error}); nothing was marked loaded." + return f"[yellow]Load ended without a status[/yellow] ({elapsed:.2f}s)." # ── Click command (for hey_egeria) ──────────────────────────────────────────── @click.command("load-report-specs") @click.option("--force", is_flag=True, default=False, - help="Bypass the TTL cache and reload from Egeria unconditionally.") + help="Ignored (it applied to the legacy loader's cache); a sync reads Egeria each time.") @click.option("--ttl", "ttl_seconds", default=None, type=int, - help="Override cache TTL for this call only (seconds). " - f"Default: PYEGERIA_REPORT_SPECS_CACHE_TTL env var or {_EGERIA_SPECS_CACHE_TTL}s.") + help="Ignored (it applied to the legacy loader's cache).") @click.pass_context def load_report_specs_cmd(ctx, force: bool, ttl_seconds: int | None) -> None: - """Refresh the report-spec registry from Egeria (ReportTypes and QuestionSpecs).""" + """DEPRECATED: syncs the report specs held in Egeria into the spec cache (use `pyegeria report-specs sync`).""" c = ctx.obj _run( view_server=c.view_server, @@ -121,13 +130,13 @@ def load_report_specs_cmd(ctx, force: bool, ttl_seconds: int | None) -> None: def main() -> None: parser = argparse.ArgumentParser( - description="Refresh the report-spec registry from Egeria." + description="DEPRECATED: sync the report specs held in Egeria into the spec cache " + "(use `pyegeria report-specs sync`)." ) parser.add_argument("--force", action="store_true", - help="Bypass the TTL cache and reload unconditionally.") + help="Ignored (it applied to the legacy loader's cache).") parser.add_argument("--ttl", dest="ttl_seconds", type=int, default=None, - help=f"Override cache TTL (seconds). " - f"Default: PYEGERIA_REPORT_SPECS_CACHE_TTL env var or {_EGERIA_SPECS_CACHE_TTL}s.") + help="Ignored (it applied to the legacy loader's cache).") parser.add_argument("--server", dest="server", default=app_config.egeria_view_server, help="Egeria view server name") diff --git a/commands/tech/list_element_register.py b/commands/tech/list_element_register.py index 230c3a16..9991adc1 100644 --- a/commands/tech/list_element_register.py +++ b/commands/tech/list_element_register.py @@ -75,23 +75,23 @@ def _display_name(properties: dict, qualified_name: str) -> str: return qualified_name or "---" -def _fetch_all_elements(c_client: EgeriaTech, om_type: str, page_size: int) -> list[dict]: - """Page through get_elements until an empty page comes back. - - Per Egeria's own paging contract, a short page does not mean the last - page -- only an empty result does -- so this keeps asking until that - happens rather than stopping the moment a page comes back under - page_size. +def _fetch_all_elements(c_client: EgeriaTech, om_type: str, page_size: int, + notes: list[str] | None = None) -> list[dict]: + """Every element of `om_type`, read through pyegeria's pager (pager step 4; a suspect path). + + Per Egeria's paging contract a short page is not the last page and an empty one is a filtered page, not + the end: only null ends the read, and on this path even null isn't proof (the pager probes past it). + When the end isn't confirmed, `notes` (if given) gets "Showing at least N elements: ...", so the + register's counts are never shown as totals. """ - all_elements: list[dict] = [] - start_from = 0 - while True: - page = c_client.get_elements(om_type, start_from=start_from, page_size=page_size, output_format="JSON") - if not isinstance(page, list) or len(page) == 0: - break - all_elements.extend(page) - start_from += page_size - return all_elements + from pyegeria.core.paging import at_least_note, log_end, page_all, paging_for + paged = page_all(lambda start, size: c_client.get_elements(om_type, start_from=start, page_size=size, + output_format="JSON"), + fetch_page_size=page_size, **paging_for("get_elements")) + log_end("list_element_register._fetch_all_elements", paged) + if notes is not None and (words := at_least_note(paged, "elements")): + notes.append(words) + return [e for e in paged.items if isinstance(e, dict)] def element_register( @@ -115,7 +115,9 @@ def element_register( sys.exit(1) try: - raw_elements = _fetch_all_elements(c_client, om_type, page_size) + read_notes: list[str] = [] + raw_elements = _fetch_all_elements(c_client, om_type, page_size, read_notes) + at_least = "≥" if read_notes else "" # pager step 4: counts are "at least" unless the end was confirmed rows = [] for element in raw_elements: @@ -154,9 +156,9 @@ def element_register( stats = Table.grid(padding=(0, 3)) stats.add_row( - Text(str(len(rows)), style="bold white"), - Text(str(len(groups)), style="bold white"), - Text(str(len(distinct_classifications)), style="bold white"), + Text(f"{at_least}{len(rows)}", style="bold white"), + Text(f"{at_least}{len(groups)}", style="bold white"), + Text(f"{at_least}{len(distinct_classifications)}", style="bold white"), ) stats.add_row( Text("ELEMENTS", style="dim"), @@ -169,7 +171,8 @@ def element_register( header_lines.append(f"Search: '{search}'") if classification: header_lines.append(f"Classification filter: '{classification}'") - header_lines.append(f"{len(rows)} of {len(raw_elements)} shown · {time.asctime()}") + header_lines.append(f"{len(rows)} of {at_least}{len(raw_elements)} shown · {time.asctime()}") + header_lines.extend(read_notes) panel = Panel( Group(stats, Padding(Text("\n".join(header_lines), style="dim"), (1, 0, 0, 0))), diff --git a/docs/design/report_spec_migration_design.md b/docs/design/report_spec_migration_design.md index 253cde56..24e5b070 100644 --- a/docs/design/report_spec_migration_design.md +++ b/docs/design/report_spec_migration_design.md @@ -1,5 +1,13 @@ # Design: Migrating Report Specs to Egeria +> **Superseded loader (2026-11-14).** `load_egeria_report_specs()`, the loader this document describes, is the +> legacy path for the May-2026 `ReportType::` collections. It is **deprecated and will be removed in pyegeria 7 +> (expected early January 2027)**. Egeria-held report specs are `ReportSpec::` elements, and they load through +> the spec cache: `pyegeria.view.egeria_spec_cache.sync_report_specs` (`pyegeria report-specs sync`), the system of +> record. It reads both markers and serves the registry's EGERIA tier, the consumer index and the authoring API. +> pyegeria's own callers have moved: the MCP server's startup and the `load_report_specs` / `hey_egeria ... +> load-report-specs` command now run the sync. The rest of this document is the May-2026 design, kept as history. + **Status:** Phase 1 (§6) shipped and complete — see "Migration status" below. Phase 2 (`formats`/`action`/`target_type`/`family`) not started. **Date:** 2026-05-17 diff --git a/docs/release-notes/next.md b/docs/release-notes/next.md new file mode 100644 index 00000000..a494d605 --- /dev/null +++ b/docs/release-notes/next.md @@ -0,0 +1,100 @@ +# pyegeria (next release, unreleased) + +Notes collected on `release/next` for the next release. The version number is set when it's cut. + +## New + +- **`NO_PASSWORD`: opt out of the configured default password** (`from pyegeria.core._no_password import + NO_PASSWORD`). Pass `user_pwd=NO_PASSWORD` to any client (`EgeriaTech`, `EgeriaCat`, `Egeria`, `EgeriaConfig`, the + OMVS classes, `BaseServerClient`, `BasePlatformClient`), typically with a bearer token for a signed-in user. The + client then holds no password (`user_pwd is None`, `use_default_password is False`), and so does every sub-client + `EgeriaTech` builds and every helper client pyegeria builds for it. The configured password + (`settings.User_Profile.user_pwd`) is never used, so when the token expires the request fails with a 401 + (`PyegeriaUnauthorizedException`) instead of the one auto-retry signing in again as the configured (demo) user. + Every client pyegeria builds to act for another (sub-clients and helpers) now goes through one + `pyegeria.core._helper_client.helper_client`: the caller's bearer token set once (exactly one `Bearer ` prefix), and + the caller's own password or none, never the configured default, even when the token arrived through + `set_bearer_token`. + This replaces the password scrub and `_get_subclient` patch that token-only callers needed. **`user_pwd=None` is + unchanged**: it still takes the configured password, which CLIs rely on. +- **`AuthoringRefused.answer`** (pyegeria.authoring): every refusal now carries a machine-readable `answer`, which is + `"absent"` (Egeria holds no such artifact that you can see, said only on a confirmed end), `"unknown"` (whether + Egeria holds it couldn't be told; never to be read as "none") or `"refused"` (anything else, the default). + `ANSWER_ABSENT`, `ANSWER_UNKNOWN` and `ANSWER_REFUSED` are exported. Branch on `answer`, not on the words. It's + backward compatible: `AuthoringRefused` is still a `ValueError`, and its words are unchanged. + +- **`pyegeria.authoring.can_edit(kind, ref, client)` / `async_can_edit`** returns `{"answer": "yes"|"no"|"unknown", + "why"}`, read as the user. Egeria has no edit-rights API, so: + - **yes:** your own private draft, or a held spec, Report or sheet you created (`versions.createdBy`) or own + (Ownership names your user id); + - **no:** a built-in spec, or nothing in Egeria you can read; + - **unknown:** everything else, including a failed read. Callers hide an edit link on unknown, and the authoring + API still checks when the artifact is opened and published. +- **`pyegeria.authoring.used_by(kind, label, client)` / `async_used_by`** answers "what uses this spec, Report or + sheet?" as `{"items": [{"kind", "name", "qualified_name", "via"}], "complete", "notes"}`. Every read is live, as + the user (no shared index, no cache): + - a spec's Reports (by their `reportSpec`) and its consumer folders; + - a Report's sheets (DataSetContent); + - a sheet's consumer folders. + + `via` is one of the closed set `VIA_KEYS` (`reportSpec`, `CollectionMembership`, `DataSetContent`): machine keys + for the caller to put into words. `complete` is True only when every read reached a confirmed end. An empty answer + that isn't complete is "nothing found in what Egeria returned", never "used by none". + + The artifact is resolved first, through Egeria's exact unique-name read. If it can't be resolved, `used_by` raises + `AuthoringRefused` with `answer` "absent" (only on Egeria's own not-found) or "unknown". A built-in spec that Egeria + doesn't hold still has its Reports listed. A Report's `reportSpec` matches the label exactly, case-sensitively, + after one `ReportSpec::` prefix is stripped; the legacy `ReportType::` prefix is also stripped, until pyegeria 7. + Call it on demand, when the user asks. +- **Paging:** `get_elements` (read by type) is `findMetadataElements` underneath, so it's now NULL_NOT_END and + gate-trusted on a #9453 platform like the other find paths. The dashboard sheet list passes its client, so on + such a platform it can be confirmed complete. + +## Planned for pyegeria 7 (expected early January 2027) + +- **No default password, and no built-in identity outside a named demo profile.** pyegeria 7 drops the quickstart + defaults (`erinoverview` / `secret`): a client gets credentials only from its arguments, the environment, + config.json, or a demo profile named explicitly. + +## Deprecations + +- **`pyegeria.view.base_report_formats.load_egeria_report_specs` is deprecated and will be removed in pyegeria 7 + (expected early January 2027).** It is the legacy loader for the May-2026 `ReportType::` collections. Egeria-held + report specs are `ReportSpec::` elements, and they load through the spec cache: + `pyegeria.view.egeria_spec_cache.sync_report_specs` (the system of record, which reads both markers and serves the + consumer index and the authoring API). The legacy loader raises a `DeprecationWarning` once per process. Until it + is removed it keeps working, unchanged in what it searches. It now pages through Egeria's results, backs off its + retries after a read Egeria didn't confirm complete, and reports an empty result as "No legacy ReportType + collections found (Egeria-held specs load through the spec cache)". + +## Fixes (spec cache) + +- **The EGERIA report-spec tier is cached per identity** (`pyegeria.view.egeria_spec_cache`). It used to be held in + one shared slot, which a server with threaded, per-request identities (the Portal) could race: + - A reload for a new user marked the slot as that user's before its disk load finished, while the slot still held + the previous user's specs. + - A concurrent request for the new user in that window got the previous user's tier, private specs included, and + `egeria_tier_info` could return that user's provenance. + + Now each identity has its own entry, held in a bounded LRU (64 entries). An entry is built whole and swapped in + with one locked assignment. `egeria_tier_info`, and the name check used while specs load, read only the caller's + own entry. +- **`spec_identity(None)` now means "explicitly no one"**: no private cache loads and no fetched copy is served, + whether or not the process is declared multi-user. Before, None was read as "not set" and fell back to the + process's own identity. An empty identity means no one too. + +## Fixes + +- **The exact unique-name read no longer signs in with the settings default password** (`pyegeria.core.unique_name`). + When the caller has no unique-name read of its own (a `CollectionManager`, say), pyegeria builds a + `ClassificationExplorer` on the same platform. It now acts exactly as the caller: + - The explorer gets the caller's token and is built with `user_pwd=NO_PASSWORD`, so it never holds a password. Before, a caller that had cleared its password (as Studio + and EA do) gave the explorer the settings default. A 401 auto-retry could then sign in again as that default user. + - With no token, the explorer uses only `carried_password(client)`: a password the caller itself holds, or the + caller's own `NO_PASSWORD` opt-out. If the caller has neither, the answer is + "unknown" and Egeria isn't called. + - The token is shared bare. Before, the `"Bearer ..."` header value was set as the token, which gave `"Bearer + Bearer ..."`. + - A caller without real `view_server` and `platform_url` strings (a fake, or anything that isn't a client) gets + "unknown". No client is built. + - The explorer is closed after the read. diff --git a/docs/search-and-get-filters.md b/docs/search-and-get-filters.md index 45a29b3b..0e6c5985 100644 --- a/docs/search-and-get-filters.md +++ b/docs/search-and-get-filters.md @@ -209,7 +209,7 @@ Used by get_*_by_name and other list gets. Maps to `FilterRequestBody` (by name) | `anchor_guid` | `str` | `anchorGUID` | Only return elements anchored to the element with this GUID. | | `anchor_domain` | `str` | `anchorDomainName` | Only return elements whose anchor is in this domain (an open metadata type name). | | `anchor_scope_guid` | `str` | `anchorScopeGUID` | Only return elements whose anchor scope is the element with this GUID. | -| `limit_results_by_status` | `list[str]` | `limitResultsByStatus` | Only return elements whose status is one of these, e.g. `["ACTIVE", "DRAFT"]`. | +| `limit_results_by_status` | `list[str]` | `limitResultsByStatus` | Only return elements whose status is one of these, e.g. `["ACTIVE"]`; Egeria's ElementStatus: UNKNOWN, ACTIVE, INVALID, DELETED, OTHER (a content status such as DRAFT is refused). | | `sequencing_order` | `str` | `sequencingOrder` | Order of the results; `PROPERTY_ASCENDING`/`PROPERTY_DESCENDING` need `sequencing_property`. | | `sequencing_property` | `str` | `sequencingProperty` | Property name to sort on when `sequencing_order` is a PROPERTY_* order. | diff --git a/md_processing/data/compact_commands/commands_report_compact.json b/md_processing/data/compact_commands/commands_report_compact.json index 246d52cb..2f97c01a 100644 --- a/md_processing/data/compact_commands/commands_report_compact.json +++ b/md_processing/data/compact_commands/commands_report_compact.json @@ -257,26 +257,13 @@ "default_value": "['ACTIVE']", "valid_values": [ "UNKNOWN", - "DRAFT", - "PREPARED", - "PROPOSED", - "APPROVED", - "REJECTED", - "APPROVED_CONCEPT", - "UNDER_DEVELOPMENT", - "DEVELOPMENT_COMPLETE", - "APPROVED_FOR_DEPLOYMENT", - "STANDBY", "ACTIVE", - "FAILED", - "DISABLED", - "COMPLETE", - "DEPRECATED", - "OTHER", - "DELETED" + "INVALID", + "DELETED", + "OTHER" ], "existing_element": "", - "description": "One of the status values from a list of valid values.", + "description": "The element (header) statuses to limit results to: Egeria's ElementStatus, one of UNKNOWN, ACTIVE, INVALID, DELETED, OTHER. Content statuses such as DRAFT or APPROVED are not element statuses and are refused.", "generated": false, "style": "Valid Value", "user_specified": true, diff --git a/md_processing/data/generated_format_sets.json b/md_processing/data/generated_format_sets.json index 42cbe105..c4daebc5 100644 --- a/md_processing/data/generated_format_sets.json +++ b/md_processing/data/generated_format_sets.json @@ -231594,23 +231594,10 @@ "detail_spec": null, "valid_values": [ "UNKNOWN", - "DRAFT", - "PREPARED", - "PROPOSED", - "APPROVED", - "REJECTED", - "APPROVED_CONCEPT", - "UNDER_DEVELOPMENT", - "DEVELOPMENT_COMPLETE", - "APPROVED_FOR_DEPLOYMENT", - "STANDBY", "ACTIVE", - "FAILED", - "DISABLED", - "COMPLETE", - "DEPRECATED", - "OTHER", - "DELETED" + "INVALID", + "DELETED", + "OTHER" ] }, { @@ -232100,23 +232087,10 @@ "detail_spec": null, "valid_values": [ "UNKNOWN", - "DRAFT", - "PREPARED", - "PROPOSED", - "APPROVED", - "REJECTED", - "APPROVED_CONCEPT", - "UNDER_DEVELOPMENT", - "DEVELOPMENT_COMPLETE", - "APPROVED_FOR_DEPLOYMENT", - "STANDBY", "ACTIVE", - "FAILED", - "DISABLED", - "COMPLETE", - "DEPRECATED", - "OTHER", - "DELETED" + "INVALID", + "DELETED", + "OTHER" ] }, { @@ -232475,23 +232449,10 @@ "detail_spec": null, "valid_values": [ "UNKNOWN", - "DRAFT", - "PREPARED", - "PROPOSED", - "APPROVED", - "REJECTED", - "APPROVED_CONCEPT", - "UNDER_DEVELOPMENT", - "DEVELOPMENT_COMPLETE", - "APPROVED_FOR_DEPLOYMENT", - "STANDBY", "ACTIVE", - "FAILED", - "DISABLED", - "COMPLETE", - "DEPRECATED", - "OTHER", - "DELETED" + "INVALID", + "DELETED", + "OTHER" ] }, { @@ -232989,23 +232950,10 @@ "detail_spec": null, "valid_values": [ "UNKNOWN", - "DRAFT", - "PREPARED", - "PROPOSED", - "APPROVED", - "REJECTED", - "APPROVED_CONCEPT", - "UNDER_DEVELOPMENT", - "DEVELOPMENT_COMPLETE", - "APPROVED_FOR_DEPLOYMENT", - "STANDBY", "ACTIVE", - "FAILED", - "DISABLED", - "COMPLETE", - "DEPRECATED", - "OTHER", - "DELETED" + "INVALID", + "DELETED", + "OTHER" ] }, { diff --git a/md_processing/v2/dashboard_sheet.py b/md_processing/v2/dashboard_sheet.py index 57625035..54251080 100644 --- a/md_processing/v2/dashboard_sheet.py +++ b/md_processing/v2/dashboard_sheet.py @@ -51,6 +51,11 @@ PLANNED_PREFIX = "planned:" PAGE_SIZE = 25 + +def _element_key(element: Any) -> Any: + from pyegeria.core.paging import default_key + return default_key(element) + # ---------------------------------------------------------------------------- qualified names DASHBOARD_QN_PREFIX = "Dashboard::" @@ -310,28 +315,32 @@ async def _find(self, qualified_name: str) -> Optional[Dict[str, Any]]: return await find_element_by_qualified_name(self.client, qualified_name) async def _related(self, guid: Optional[str], relationship: str, start_at_end: int = 1) -> List[Dict[str, Any]]: - """Every element related to `guid` by `relationship`, paged at 25. Only an empty page ends the read - (a short page is not the last page).""" + """Every element related to `guid` by `relationship`, read through the pager (pager step 4; a + suspect path) at PAGE_SIZE. A read that reached the end (a null, then empty probes) is acted on; + one a limit stopped (capped, stalled, the empty-run cap) returns what was read with a note, so a + sheet written from it is never silently short. Each read's end is logged.""" if not guid or self._is_planned(guid): return [] - out: List[Dict[str, Any]] = [] - start = 0 - while True: + from pyegeria.core.paging import async_page_all, paging_for + + async def fetch(start, size): try: - page = await self.client.classification_manager._async_get_related_elements( - guid, relationship, start_at_end=start_at_end, start_from=start, page_size=PAGE_SIZE, + return await self.client.classification_manager._async_get_related_elements( + guid, relationship, start_at_end=start_at_end, start_from=start, page_size=size, graph_query_depth=0) except PyegeriaNotFoundException: - page = [] - items = [e for e in page if isinstance(e, dict) and e.get("elementHeader")] if isinstance(page, list) else [] - if not items: - return out - known = {self._guid(e) for e in out} - if all(self._guid(e) in known for e in items): # paging did not advance; it would never end (6.2.36) - logger.warning(f"dashboard sheet: paging of {relationship} for {guid} did not advance at {start}") - return out - out.extend(e for e in items if self._guid(e) not in known) - start += len(items) + return None # nothing related: the end + + paged = await async_page_all(fetch, fetch_page_size=PAGE_SIZE, **paging_for("get_related_elements"), + key=_element_key) + return self._read_settled(paged, f"the {relationship} links of {guid}") + + def _read_settled(self, paged, what: str) -> List[Dict[str, Any]]: + from pyegeria.core.paging import log_end + log_end(f"dashboard_sheet ({what})", paged) + if not paged.reached_end: + self._note(f"{what} weren't read to the end ({paged.note}); what was read was used") + return [e for e in paged.items if isinstance(e, dict) and e.get("elementHeader")] @staticmethod def _guid(element: Dict[str, Any]) -> str: @@ -508,45 +517,76 @@ async def ensure_ownership(self, dashboard_guid: Optional[str], qn: str, owner: # ------------------------------------------------------------ placements async def _resolve_report(self, report_name: str) -> Optional[str]: - """The GUID of the Report a placement shows: the one `Report` whose displayName is exactly - `report_name` (an exact, type-filtered lookup: 0.1 s live, where a contains-search took ~35 s), - else the Report whose qualifiedName it is. None if there - is none; two or more exact matches are a note, and no link. + """The GUID of the Report a placement shows, or None (with a note that reaches the author). + + The Architect's ruling on this lookup (2026-11-14): a placement link is a reversible relationship, so a + display-name match may be linked, but its uncertainty is said every time. + (a) A ref that looks like a qualified name ("::") is read by Egeria's unique-name read: found -> link; + Egeria's not-found -> the display-name path; any other error -> no link, with a note. + (b) A display-name ref is read through the pager (get_elements_by_property_value is NULL_NOT_END, and + trusted on a platform with the #9453 fix), with exact displayName equality among Reports: + - (step 6) on a trusted end: 1 match -> link, no note; 0 -> "There is no Report named X"; + - exactly 1 match, the read at null_then_empty_probes -> link, with a note naming its qualifiedName; + - exactly 1 match, the read stopped at a limit -> no link, with a note; + - 2 or more -> no link, with a note listing up to 5 qualifiedNames; + - 0, or unknown -> no link, with "none found in what Egeria returned" (never "no such Report"). Not Dr.Egeria's `__async_get_guid__`: it tries a unique-name lookup first, which Egeria rejects when the name is shared with an element of another type -- seen live 2026-11-09 for 'Term Definition Completeness' and 'Governance Classification Coverage', both real Reports.""" - matches: List[str] = [] - seen: set = set() - start = 0 - while True: + from pyegeria.core.paging import async_page_all, async_paging_for + + if "::" in report_name: # (a) try: - page = await self.client.classification_manager._async_get_elements_by_property_value( + found = await self.client.classification_manager._async_get_element_by_unique_name(report_name) + except PyegeriaNotFoundException: + found = None + except Exception as exc: # noqa: BLE001 -- said, and no link is written + self._note(f"The Report '{report_name}' couldn't be read ({type(exc).__name__}): no report link " + f"was written for that placement.") + return None + element = found if isinstance(found, dict) and found.get("elementHeader") else None + if element and self._type(element) == "Report": + return self._guid(element) + + async def fetch(start, size): + try: + return await self.client.classification_manager._async_get_elements_by_property_value( report_name, ["displayName"], metadata_element_type="Report", - start_from=start, page_size=PAGE_SIZE, graph_query_depth=0) + start_from=start, page_size=size, graph_query_depth=0) except PyegeriaNotFoundException: - page = [] - items = [e for e in page if isinstance(e, dict) and e.get("elementHeader")] if isinstance(page, list) else [] - if not items: - break - if all(self._guid(e) in seen for e in items): # paging did not advance; it would never end (6.2.36) - logger.warning(f"dashboard sheet: paging of Reports named {report_name!r} did not advance at {start}") - break - items = [e for e in items if self._guid(e) not in seen] - seen.update(self._guid(e) for e in items) - matches += [self._guid(e) for e in items - if (e.get("properties") or {}).get("displayName") == report_name - and self._type(e) == "Report"] - start += len(items) - if len(matches) == 1: - return matches[0] + return None # no Report by that name in this read + + paged = await async_page_all(fetch, fetch_page_size=PAGE_SIZE, + **await async_paging_for("get_elements_by_property_value", self.client), + key=_element_key) + from pyegeria.core.paging import log_end + log_end(f"dashboard_sheet (the Reports named {report_name!r})", paged) + matches = [e for e in paged.items if isinstance(e, dict) and e.get("elementHeader") + and (e.get("properties") or {}).get("displayName") == report_name and self._type(e) == "Report"] + qns = [(e.get("properties") or {}).get("qualifiedName") or self._guid(e) for e in matches] if len(matches) > 1: - self._note(f"{len(matches)} Reports are named '{report_name}' in Egeria: no report link was written " - f"for that placement. Give the placement's report a unique name.") + listed = ", ".join(f"'{q}'" for q in qns[:5]) + (" ..." if len(qns) > 5 else "") + self._note(f"{len(matches)} Reports are named '{report_name}' in Egeria ({listed}): no report link was " + f"written for that placement. Give the placement the Report's qualified name.") + return None + if len(matches) == 1 and paged.complete: # step 6: a trusted end on a #9453 platform: unique, no note + return self._guid(matches[0]) + if len(matches) == 1 and paged.end_reason == "null_then_empty_probes": + self._note(f"Linked the Report named '{report_name}' that Egeria returned ('{qns[0]}'); Egeria can't " + f"confirm it's the only one; give the Report's qualified name to be sure.") + return self._guid(matches[0]) + if len(matches) == 1: + self._note(f"One Report named '{report_name}' was found ('{qns[0]}'), but the read stopped before the " + f"end ({paged.note}): no report link was written for that placement. Give the placement the " + f"Report's qualified name.") + return None + if paged.complete: # step 6: a confirmed empty read may say "none" + self._note(f"There is no Report named '{report_name}' in Egeria: no report link was written for that " + f"placement.") return None - element = await self._find(report_name) - if element and self._type(element) == "Report": - return self._guid(element) + self._note(f"No Report named '{report_name}' was found in what Egeria returned ({paged.note or paged.end}): " + f"no report link was written for that placement.") return None async def _perspective_guid(self, label: str) -> Optional[str]: @@ -624,9 +664,10 @@ async def _sync_report_link(self, chain: SheetChain, placement: Placement, qn: s if report_guid is None and not resolved: report_guid = await self._resolve_report(placement.ref) if not report_guid: - self._note(f"Report '{placement.ref}' was not found in Egeria: placement '{placement.ref}' on sheet " - f"'{chain.sheet_name}' was written without its report link, and reads as missing until a " - f"Report with that name exists and the command is run again.") + # Never "not found in Egeria" here: on the display-name path (NULL_NOT_END) none found isn't proof + # (the lookup's own note says why no link was written). + self._note(f"Placement '{placement.ref}' on sheet '{chain.sheet_name}' was written without its report " + f"link, and reads as missing until its Report can be linked and the command is run again.") if binding["links"]: await self._detach_report(chain, qn, "its report is gone") return diff --git a/md_processing/v2/processors.py b/md_processing/v2/processors.py index 3391c739..d8e958a0 100644 --- a/md_processing/v2/processors.py +++ b/md_processing/v2/processors.py @@ -43,6 +43,12 @@ _AMBIGUOUS_SILENT = object() +def _is_not_found_error(exc: BaseException) -> bool: + """Egeria's specific not-found (pager step 5): an answer ("absent"), unlike any other lookup error.""" + from pyegeria.core._exceptions import PyegeriaNotFoundException + return isinstance(exc, PyegeriaNotFoundException) + + def _is_egeria_multiple_entities_error(exc: BaseException) -> bool: text = str(exc) return "404-002" in text or "entities where found" in text @@ -87,6 +93,15 @@ class _ParentLinksIncomplete(Exception): """The parent-end links could not all be read within the page bound (ISSUE-181).""" +def _read_words(read: Optional[dict]) -> str: + """A paged read's trace in the run result (pager step 4): positions read, end and why.""" + if not read: + return "not read" + return (f"{read.get('positions_read')} positions, end {read.get('end')} " + f"({read.get('end_reason')}); Egeria didn't confirm there were no more" + if read.get("end") != "confirmed" else f"{read.get('positions_read')} positions, end confirmed") + + class AsyncBaseCommandProcessor(ABC): """ Base class for all v2 Dr.Egeria command processors. @@ -676,7 +691,8 @@ async def _sync_parent_relationship(self, guid: str, attributes: Dict[str, Any]) except _ParentLinksIncomplete: # Adding without seeing every link could duplicate the parent link. self.add_related_result("Parent Relationship", guid=parent_guid, status="failure", - message=f"could not read every {rel_type} link; not added") + message=f"could not read every {rel_type} link; not added " + f"(read: {_read_words(self._parent_link_read)})") return body = { @@ -690,6 +706,8 @@ async def _sync_parent_relationship(self, guid: str, attributes: Dict[str, Any]) body["properties"] = new_element_properties(rel_props) await self.client.metadata_expert._async_create_related_elements(body) message = f"Linked via {rel_type}" + if (read := getattr(self, "_parent_link_read", None)) and read.get("end") != "confirmed": + message += f" (the existing links were read: {_read_words(read)})" if other_parents: message += (f"; existing {rel_type} link(s) to {', '.join(other_parents)} left in place" " (remove them explicitly to move the element)") @@ -707,7 +725,8 @@ async def _parent_links_of_type(self, guid: str, rel_type: str, parent_at_end1: Only relationships with `guid` at the child end count: end 2 when the parent is at end 1, and the other way round. Egeria filters by `startingAtEnd`, and each entry's `elementAtEnd1`, when present, is - checked as well. Pages until an empty page (a short page is not the last). + checked as well. Read through the pager (pager step 4; a suspect path): acted on when the read + reached the end; a limit (capped, stalled, the empty-run cap) raises _ParentLinksIncomplete. `_async_get_all_related_elements` returns a dict - {"startingElement", "elementList", "mermaidGraph"} - whose entries are the low-level @@ -715,28 +734,34 @@ async def _parent_links_of_type(self, guid: str, rel_type: str, parent_at_end1: element.elementGUID), not the relationshipHeader/relatedElement shape of the domain get-by-guid calls. """ + from pyegeria.core.paging import async_page_all, log_end, paging_for child_end = 2 if parent_at_end1 else 1 - start = 0 - for _ in range(self._PARENT_LINK_MAX_PAGES): + + async def fetch(start, size): page = await self.client.metadata_expert._async_get_all_related_elements( - guid, starting_at_end=child_end, start_from=start, - page_size=self._PARENT_LINK_PAGE_SIZE) + guid, starting_at_end=child_end, start_from=start, page_size=size) element_list = page.get("elementList") if isinstance(page, dict) else None - if not element_list: - return - for rel in element_list: - try: - if rel["type"]["typeName"] != rel_type: - continue - related_guid = rel["element"]["elementGUID"] - except (KeyError, TypeError): + return element_list if isinstance(element_list, list) else None # absent: null + + # Pager step 4, ruling (A) 3: a write path acts on a null-then-empty-probes end, logs and keeps the + # read's trace (method, positions_read, end, end_reason), and stops only on a limit. + paged = await async_page_all(fetch, fetch_page_size=self._PARENT_LINK_PAGE_SIZE, + max_pages=self._PARENT_LINK_MAX_PAGES, + **paging_for("get_all_related_elements")) + self._parent_link_read = log_end("processors._parent_links_of_type", paged) + if not paged.reached_end: + raise _ParentLinksIncomplete(rel_type) + for rel in paged.items: + try: + if rel["type"]["typeName"] != rel_type: continue - at_end1 = rel.get("elementAtEnd1") - if at_end1 is not None and bool(at_end1) != bool(parent_at_end1): - continue # the related element is at the child end: not a parent link - yield related_guid - start += len(element_list) - raise _ParentLinksIncomplete(rel_type) + related_guid = rel["element"]["elementGUID"] + except (KeyError, TypeError): + continue + at_end1 = rel.get("elementAtEnd1") + if at_end1 is not None and bool(at_end1) != bool(parent_at_end1): + continue # the related element is at the child end: not a parent link + yield related_guid def effective_for_lineage(self) -> bool: """Whether this command's requests are sent with forLineage=true. @@ -1096,7 +1121,15 @@ async def _execute(self) -> Dict[str, Any]: # the attribute appeared to succeed while quietly doing nothing. attr_data["exists"] = False attr_data["valid"] = False # Treat as invalid to block execution - msg = f"Referenced element(s) {genuinely_failed} for attribute '{attr_name}' not found." + unknown_items = [i for i in genuinely_failed if self._lookup_was_unknown(i)] + if unknown_items: # pager step 6: unknown is never "not found" + msg = (f"Couldn't tell whether referenced element(s) {unknown_items} for attribute " + f"'{attr_name}' exist in Egeria (the lookup failed); nothing was written.") + others = [i for i in genuinely_failed if i not in unknown_items] + if others: + msg += f" Referenced element(s) {others} not found." + else: + msg = f"Referenced element(s) {genuinely_failed} for attribute '{attr_name}' not found." if attr_data.get("errors") is None: attr_data["errors"] = [] attr_data["errors"].append(msg) logger.error(msg) @@ -1126,7 +1159,11 @@ async def _execute(self) -> Dict[str, Any]: # If it's a candidate ref and we couldn't resolve it, mark as not found attr_data["exists"] = False attr_data["valid"] = False # Treat as invalid to block execution - msg = f"Referenced element '{val}' for attribute '{attr_name}' not found." + if self._lookup_was_unknown(val): # pager step 6: unknown is never "not found" + msg = (f"Couldn't tell whether referenced element '{val}' for attribute '{attr_name}' " + f"exists in Egeria (the lookup failed); nothing was written.") + else: + msg = f"Referenced element '{val}' for attribute '{attr_name}' not found." if attr_data.get("errors") is None: attr_data["errors"] = [] attr_data["errors"].append(msg) logger.error(msg) @@ -1352,6 +1389,27 @@ async def _execute(self) -> Dict[str, Any]: "found": self.parsed_output.get("exists", False) } + # Pager step 5 (the exists-lookups ruling, 6): a lookup that failed with an error other than Egeria's + # not-found can't say the target is absent, so a command whose target wasn't found doesn't write: a + # transport, auth or timeout error must never make Create-or-Update create a duplicate. + unknown = self.__dict__.get("_lookups_unknown") or [] + if unknown and self.supports_target_element_lookup() and not self.as_is_element: + msg = f"{'; '.join(unknown)}, so nothing was created or changed. Try again." + logger.error(f"Execution blocked for {self.command.verb} {self.command.object_type}: {msg}") + return { + "output": f"{self.command.raw_block}\n\n> ❌ **Execution Blocked**\n> - {msg}", + "analysis": analysis, + "status": "failure", + "message": f"Execution blocked: {msg}", + "verb": self.command.verb, + "object_type": self.canonical_object_type, + "markdown_object_type": self.markdown_object_type, + "display_name": self.parsed_output.get("display_name"), + "qualified_name": self.parsed_output.get("qualified_name"), + "found": False, + "errors": unknown, + } + try: if self.context.get("debug"): print( @@ -1767,11 +1825,13 @@ def _cache_if_not_found(key: tuple, value: Any) -> None: res = cached1 else: try: - res = await self.client.__async_get_guid__(qualified_name=name_or_guid, display_name=name_or_guid, property_name="displayName", tech_type=tech_type or None) + res = await self.client.__async_get_guid__(qualified_name=name_or_guid, display_name=name_or_guid, property_name="displayName", tech_type=tech_type or None, strict_errors=True) _cache_if_not_found(pass1_key, res) except PyegeriaException as e: + if _is_not_found_error(e): + res = None # Egeria's not-found: absent # Catch multiple matches error - if "Multiple elements found" in str(e): + elif "Multiple elements found" in str(e): query_cache[pass1_key] = _AMBIGUOUS _report_ambiguous("Pass 1") return None @@ -1794,7 +1854,9 @@ def _cache_if_not_found(key: tuple, value: Any) -> None: logger.debug(f"SDK strict lookup (Pass 1) failed for '{name_or_guid}': {e}") if self.context.get("directive") == "validate": print_basic_exception(e) - # Not cached -- unexpected error, not a real answer. + # Not cached -- unexpected error, not a real answer. Pager step 5: nor "absent" -- + # recorded as unknown, so a Create-or-Update never creates on it. + self._note_lookup_unknown(name_or_guid, e) # Pass 2: If no result (or if type was invalid), try WITHOUT type constraint is_not_found = not res or (isinstance(res, str) and (res.startswith("No ") or " found" in res)) @@ -1813,10 +1875,12 @@ def _cache_if_not_found(key: tuple, value: Any) -> None: res = cached2 else: try: - res = await self.client.__async_get_guid__(qualified_name=name_or_guid, display_name=name_or_guid, property_name="displayName") + res = await self.client.__async_get_guid__(qualified_name=name_or_guid, display_name=name_or_guid, property_name="displayName", strict_errors=True) _cache_if_not_found(pass2_key, res) except PyegeriaException as e: - if "Multiple elements found" in str(e): + if _is_not_found_error(e): + res = None # Egeria's not-found: absent + elif "Multiple elements found" in str(e): query_cache[pass2_key] = _AMBIGUOUS _report_ambiguous("Pass 2") return None @@ -1824,9 +1888,11 @@ def _cache_if_not_found(key: tuple, value: Any) -> None: query_cache[pass2_key] = _AMBIGUOUS_SILENT logger.debug(f"resolve_element_guid: '{name_or_guid}' is ambiguous (Pass 2): {e}") return None - logger.debug(f"SDK strict lookup (Pass 2) failed for '{name_or_guid}': {e}") - if self.context.get("directive") == "validate": - print_basic_exception(e) + else: + logger.debug(f"SDK strict lookup (Pass 2) failed for '{name_or_guid}': {e}") + if self.context.get("directive") == "validate": + print_basic_exception(e) + self._note_lookup_unknown(name_or_guid, e) # pager step 5: unknown, never absent # Ensure it's not a "not found" indicator string if res and isinstance(res, str) and not res.startswith("No ") and " found" not in res and not res.startswith("(Planned:"): @@ -1835,9 +1901,30 @@ def _cache_if_not_found(key: tuple, value: Any) -> None: except Exception as e: logger.debug(f"resolve_element_guid: Unexpected error resolving '{name_or_guid}': {e}") - + self._note_lookup_unknown(name_or_guid, e) # pager step 5: unknown, never absent + return None + def _lookup_was_unknown(self, name: str) -> bool: + """Whether a lookup of `name` failed with an error other than Egeria's not-found (pager step 6: then a + reference is "couldn't tell", never "not found").""" + prefix = f"couldn't tell whether '{name}' exists" + return any(w.startswith(prefix) for w in self.__dict__.get("_lookups_unknown") or []) + + def _note_lookup_unknown(self, name: str, exc: BaseException) -> None: + """Pager step 5 (the exists-lookups ruling, 6): a lookup that failed with an error that isn't Egeria's + not-found can't tell whether `name` exists. It's recorded, and execute() blocks a write whose target + wasn't found while any lookup is unknown, so a transport, auth or timeout error never becomes a create.""" + if _is_not_found_error(exc): + return # Egeria's not-found is an answer: absent + if getattr(self, "client", None) is None: + return # no Egeria connection (a local-only store): there is no Egeria lookup to be unknown + words = f"couldn't tell whether '{name}' exists in Egeria ({type(exc).__name__})" + unknown = self.__dict__.setdefault("_lookups_unknown", []) + if words not in unknown: + unknown.append(words) + logger.warning(f"resolve_element_guid: {words}") + async def _extract_memberships_async(self, async_get_fn, guid: str) -> dict: """ Async helper that fetches an element by GUID and extracts its diff --git a/my_egeria/my_egeria/CommunitiesScreen.py b/my_egeria/my_egeria/CommunitiesScreen.py index 1b36596a..e9e7b834 100644 --- a/my_egeria/my_egeria/CommunitiesScreen.py +++ b/my_egeria/my_egeria/CommunitiesScreen.py @@ -12,6 +12,7 @@ from textual.widgets import DataTable, Header, Static, Footer, Input, Button, Select, TextArea from CommunityScreen import CommunityScreen +from paging_support import is_complete, none_words from profile_utils import set_screen_heading # The "Linked Through" value of a membership held by the user's own profile, which they can leave @@ -224,8 +225,14 @@ def _search(self) -> None: if row[3] not in mine: table.add_row(*row, key=row[3]) if table.row_count == 0: - self.notify(f"No other communities found matching {self.search_string}", timeout=5, - severity="warning") + if is_complete(found): + self.notify(f"No other communities found matching {self.search_string}", timeout=5, + severity="warning") + else: + self.notify(none_words("other communities", complete=False, matching=self.search_string), + timeout=10, severity="warning") + elif getattr(found, "note", None): + self.notify(found.note, timeout=10, severity="warning") def _reload(self) -> None: rows = self.app.list_my_communities() diff --git a/my_egeria/my_egeria/CommunityScreen.py b/my_egeria/my_egeria/CommunityScreen.py index 27e476d5..825b305f 100644 --- a/my_egeria/my_egeria/CommunityScreen.py +++ b/my_egeria/my_egeria/CommunityScreen.py @@ -14,6 +14,7 @@ from textual.widgets import DataTable, Header, Static, Footer, Input, Button, Select, TextArea from pyegeria import COMMENT_TYPES +from paging_support import is_complete, none_words from profile_utils import set_screen_heading @@ -213,6 +214,7 @@ def _load_posts(self) -> None: return table = self.query_one("#community_posts_table", DataTable) table.clear() + self.posts_complete = is_complete(posts) self.post_text = {} for title, author, posted, text, guid in posts: table.add_row(title, author, posted, guid, key=guid) @@ -233,8 +235,10 @@ def _show_post_text(self) -> None: if table.row_count: guid = table.get_row_at(table.cursor_row)[3] text = self.post_text.get(guid, "") - else: + elif getattr(self, "posts_complete", True): text = "[dim]No posts yet - be the first to post[/dim]" + else: + text = f"[dim]{none_words('posts', complete=False)}[/dim]" self.query_one("#community_post_text", Static).update(text) @on(DataTable.RowHighlighted, "#community_posts_table") diff --git a/my_egeria/my_egeria/FeedbackScreens.py b/my_egeria/my_egeria/FeedbackScreens.py index df10401f..93c8b8c0 100644 --- a/my_egeria/my_egeria/FeedbackScreens.py +++ b/my_egeria/my_egeria/FeedbackScreens.py @@ -14,6 +14,7 @@ from textual.containers import ScrollableContainer, Horizontal from textual.screen import ModalScreen from textual.widgets import Header, Static, Footer, Button, DataTable, Select, TextArea +from paging_support import count_text, is_complete, none_words from profile_utils import set_screen_heading FEEDBACK_CATEGORIES = ["Bug", "Suggestion", "Question", "Praise", "Other"] @@ -107,15 +108,19 @@ class FeedbackLogScreen(ModalScreen): def __init__(self, feedback_entries: list[dict[str, Any]], *args, **kwargs): super().__init__(*args, **kwargs) self.feedback_entries = feedback_entries or [] + # False when Egeria didn't confirm these are all the entries (paging_support.PagedRows) + self.complete = is_complete(feedback_entries) + self.count_text = count_text(feedback_entries if feedback_entries is not None else []) def on_mount(self): - set_screen_heading(self, f"Feedback Log ({len(self.feedback_entries)} entries)") + set_screen_heading(self, f"Feedback Log ({self.count_text} entries)") table = self.query_one("#feedback_log_table", DataTable) table.add_columns("Date", "Submitted By", "Category", "Screen", "Feedback") table.zebra_stripes = True table.cursor_type = "row" if not self.feedback_entries: - table.add_row("", "", "", "", "No feedback has been recorded yet") + table.add_row("", "", "", "", "No feedback has been recorded yet" if self.complete + else none_words("feedback entries", complete=False)) for entry in self.feedback_entries: table.add_row( str(entry.get("date", "")), diff --git a/my_egeria/my_egeria/MyBookMarksScreen.py b/my_egeria/my_egeria/MyBookMarksScreen.py index 542b78cd..c5cfdff2 100644 --- a/my_egeria/my_egeria/MyBookMarksScreen.py +++ b/my_egeria/my_egeria/MyBookMarksScreen.py @@ -10,6 +10,7 @@ from textual.containers import ScrollableContainer, Container, Horizontal from textual.screen import ModalScreen from textual.widgets import DataTable, Header, Static, Footer, Input, Button +from paging_support import is_complete, none_words from profile_utils import set_screen_heading @@ -33,6 +34,8 @@ def __init__(self, my_bookmarks, *args, target_guid: str | None = None, **kwargs super().__init__(id="bookmark_screen", *args, **kwargs) # Rows of (name, type, GUID) self.my_bookmarks_data = list(my_bookmarks or []) + # False when Egeria didn't confirm these are all of them (see paging_support.PagedRows) + self.my_bookmarks_complete = is_complete(my_bookmarks) # GUID of the row highlighted when the screen was opened, if any, used to pre-fill a new bookmark self.target_guid = target_guid or "" @@ -58,12 +61,16 @@ def _fill_table(self) -> None: for name, element_type, guid in self.my_bookmarks_data: table.add_row(name, element_type, guid, key=guid) if not self.my_bookmarks_data: - self.notify("You have no bookmarks yet - press Ctrl+N to add one", timeout=5) + if self.my_bookmarks_complete: + self.notify("You have no bookmarks yet - press Ctrl+N to add one", timeout=5) + else: + self.notify(none_words("bookmarks", complete=False), timeout=10, severity="warning") def _reload(self) -> None: rows = self.app.list_my_bookmarks() if rows is not None: self.my_bookmarks_data = rows + self.my_bookmarks_complete = is_complete(rows) self._fill_table() def action_quit(self) -> None: diff --git a/my_egeria/my_egeria/MyNetworkScreen.py b/my_egeria/my_egeria/MyNetworkScreen.py index 1e9f1e12..15a1d903 100644 --- a/my_egeria/my_egeria/MyNetworkScreen.py +++ b/my_egeria/my_egeria/MyNetworkScreen.py @@ -10,6 +10,7 @@ from textual.containers import ScrollableContainer, Container, Horizontal from textual.screen import ModalScreen from textual.widgets import DataTable, Header, Static, Footer, Input, Button +from paging_support import is_complete, none_words from profile_utils import set_screen_heading @@ -75,7 +76,7 @@ class MyNetworkScreen(ModalScreen): def __init__(self, my_peers, *args, **kwargs): super().__init__(id="my_network_screen", *args, **kwargs) # Rows of (qualified name, display name, full name, pronouns, GUID, description) - self.my_peers_data = list(my_peers or []) + self.my_peers_data = my_peers if isinstance(my_peers, list) else list(my_peers or []) # keeps PagedRows.complete # (display name, GUID) of the person picked from the search results to add self.selected_person = None # GUID of the peer whose details are currently displayed @@ -129,7 +130,10 @@ def _fill_table(self) -> None: for row in self.my_peers_data: table.add_row(*row, key=row[4]) if not self.my_peers_data: - self.notify("There is no-one in your network yet - search for someone to add them", timeout=5) + if is_complete(self.my_peers_data): + self.notify("There is no-one in your network yet - search for someone to add them", timeout=5) + else: + self.notify(none_words("people in your network", complete=False), timeout=10, severity="warning") def _reload(self) -> None: rows = self.app.list_my_peers() @@ -228,8 +232,14 @@ def handle_people_search(self, event: Input.Submitted) -> None: if guid not in in_network: table.add_row(name, guid, qualified_name, key=guid) if table.row_count == 0: - self.notify(f"No-one new found matching {search_string}", timeout=5, severity="warning") + if is_complete(found): + self.notify(f"No-one new found matching {search_string}", timeout=5, severity="warning") + else: + self.notify(none_words("new people", complete=False, matching=search_string), timeout=10, + severity="warning") else: + if getattr(found, "note", None): + self.notify(found.note, timeout=10, severity="warning") self.notify("Select a person to add them to your network", timeout=5) table.focus() diff --git a/my_egeria/my_egeria/SearchForTermScreen.py b/my_egeria/my_egeria/SearchForTermScreen.py index 8aef61b1..13c5b31a 100644 --- a/my_egeria/my_egeria/SearchForTermScreen.py +++ b/my_egeria/my_egeria/SearchForTermScreen.py @@ -13,6 +13,7 @@ from textual.screen import ModalScreen from textual.widgets import Header, Footer, Static, Input, Placeholder, Button, Markdown from pyegeria import load_app_config, settings, PyegeriaException, exec_report_spec +from paging_support import none_words, report_empty_words, report_end_complete from profile_utils import set_screen_heading @@ -110,10 +111,14 @@ def search_for_the_term(self, search_term): user_pass=self.password, ) if isinstance(self.search_result, dict): if self.search_result.get("kind") == "text": - if self.search_result.get("mimeType") == "text/markdown": + # the executor's key is "mime" ("mimeType" was never sent, so every found term read as none) + if (self.search_result.get("mime") or self.search_result.get("mimeType")) == "text/markdown": self.search_result = self.search_result.get("content") else: self.search_result = "# No Markdown content found for this glossary term" + elif self.search_result.get("kind") == "empty": + # "none" only when Egeria confirmed the end; otherwise the search may have stopped short + self.search_result = f"# {report_empty_words('glossary terms', self.search_result, search_term)}" else: self.search_result = "# No Markdown content found for this glossary term" else: diff --git a/my_egeria/my_egeria/SelectionOverviewScreen.py b/my_egeria/my_egeria/SelectionOverviewScreen.py index 617778f2..1ec97c8b 100644 --- a/my_egeria/my_egeria/SelectionOverviewScreen.py +++ b/my_egeria/my_egeria/SelectionOverviewScreen.py @@ -17,6 +17,7 @@ from textual.widgets import Tree, Header, Static, Placeholder, Footer, TextArea, Markdown from StatusScreen import StatusScreen +from paging_support import report_empty_words from profile_utils import set_screen_heading @@ -168,6 +169,8 @@ def display_selected_term_details(self, term_GUID) -> Any: self.glossary_term_data = self.glossary_term_data.get("content") else: self.glossary_term_data = "No Markdown content found for this glossary term" + elif self.glossary_term_data.get("kind") == "empty": + self.glossary_term_data = report_empty_words("glossary term", self.glossary_term_data, self.term_GUID) else: self.glossary_term_data = "No Markdown content found for this glossary term" else: @@ -214,7 +217,7 @@ def display_selected_digital_product(self, digital_product_GUID) -> Any: else: # self.log(f"Digital product data returned for: {self.digital_product_GUID}") if isinstance(self.digital_product_data, dict) and self.digital_product_data.get("kind") == "empty": - self.digital_product_data = "# No content found for this digital product" + self.digital_product_data = (f"# {report_empty_words('digital product', self.digital_product_data, self.digital_product_GUID)}") elif isinstance(self.digital_product_data, dict) and self.digital_product_data.get("kind") == "text": if self.digital_product_data.get("mime") == "text/markdown": self.digital_product_data = self.digital_product_data.get("content") @@ -257,7 +260,10 @@ def display_selected_data_dictionary(self, data_dictionary_GUID) -> Any: container.mount(Static(f"No data dictionary data returned for GUID: {self.data_dictionary_GUID}")) else: # self.log(f"Data dictionary data returned for GUID: {self.data_dictionary_GUID}") - if isinstance(self.data_dictionary_data, dict) and self.data_dictionary_data.get("kind") == "text": + if isinstance(self.data_dictionary_data, dict) and self.data_dictionary_data.get("kind") == "empty": + self.data_dictionary_data = report_empty_words("data dictionary", self.data_dictionary_data, + self.data_dictionary_GUID) + elif isinstance(self.data_dictionary_data, dict) and self.data_dictionary_data.get("kind") == "text": if self.data_dictionary_data.get("mime") == "text/markdown": self.data_dictionary_data = self.data_dictionary_data.get("content") else: @@ -292,6 +298,9 @@ def display_selected_business_domain(self, business_domain_GUID) -> int: if not self.business_domain_data or self.business_domain_data == []: # self.log(f"No business domain data returned for GUID: {self.business_domain_GUID}") container.mount(Static(f"No business domain data returned for GUID: {self.business_domain_GUID}")) + elif isinstance(self.business_domain_data, dict) and self.business_domain_data.get("kind") == "empty": + container.mount(Static(report_empty_words("business domain", self.business_domain_data, + self.business_domain_GUID))) else: # self.log(f"Business domain data returned for GUID: {self.business_domain_GUID}") text_area = TextArea(f"Business domain data returned for GUID: {self.business_domain_GUID}", @@ -325,6 +334,9 @@ def display_selected_data_specification(self, data_specification_GUID) -> int: if not self.data_specification_data or self.data_specification_data == []: # self.log(f"No data specification data returned for qualified name: {self.data_specification_qualified_name}") container.mount(Static(f"No data specification data returned for qualified name: {self.data_specification_qualified_name}")) + elif isinstance(self.data_specification_data, dict) and self.data_specification_data.get("kind") == "empty": + container.mount(Static(report_empty_words("data specification", self.data_specification_data, + self.data_specification_qualified_name))) else: # self.log(f"Data specification data returned for qualified name: {self.data_specification_qualified_name}") text_area = TextArea(f"Data specification data returned for qualified name: {self.data_specification_qualified_name}", @@ -352,6 +364,7 @@ def display_selected_root_collection(self, root_collection_GUID) -> int: except PyegeriaException as e: # self.log(f"Error retrieving root collection data: {e}") self.collection_data = [] + self.root_collection_data = [] # never show an earlier selection's details after a failed read container = self.query_one("#data_details_placeholder_container") container.remove_children() @@ -361,7 +374,10 @@ def display_selected_root_collection(self, root_collection_GUID) -> int: container.mount(Static(f"No Root collection data returned for GUID: {self.root_collection_GUID}")) else: # self.log(f"Root collection data returned for GUID: {self.root_collection_GUID}") - if isinstance(self.root_collection_data, dict) and self.root_collection_data.get("kind") == "text": + if isinstance(self.root_collection_data, dict) and self.root_collection_data.get("kind") == "empty": + self.root_collection_data = report_empty_words("root collection", self.root_collection_data, + self.root_collection_GUID) + elif isinstance(self.root_collection_data, dict) and self.root_collection_data.get("kind") == "text": if self.root_collection_data.get("mime") == "text/markdown": self.root_collection_data = self.root_collection_data.get("content") else: diff --git a/my_egeria/my_egeria/ShowCommentsScreen.py b/my_egeria/my_egeria/ShowCommentsScreen.py index 0eddd851..b0f4b132 100644 --- a/my_egeria/my_egeria/ShowCommentsScreen.py +++ b/my_egeria/my_egeria/ShowCommentsScreen.py @@ -15,6 +15,7 @@ from textual.widgets import DataTable, Header, Static, Footer, Input, Button from pyegeria import PyegeriaException, EgeriaTech, exec_report_spec, load_app_config, settings +from paging_support import none_words, report_empty_words, report_end_complete from profile_utils import set_screen_heading @@ -94,7 +95,8 @@ def load_comments(self, element_guid: str) -> None: user_pass=self.user_password, ) except PyegeriaException as e: - self.notify(f"No comments found: {str(e)}", severity="warning") + # a failed read is not "no comments" + self.notify(f"Couldn't retrieve the comments: {str(e)}", severity="error") return # self.log(f"comments_list: {comments_list}") @@ -116,7 +118,12 @@ def load_comments(self, element_guid: str) -> None: comment_table.add_row(str(comment), "", "", "") if comment_table.row_count == 0: - if self.selected_comment_guid: + if not report_end_complete(comments_list): + # Egeria didn't confirm the end, so "has no comments" can't be said + what = "responses to this comment" if self.selected_comment_guid else "comments on the selected element" + self.notify(f"{none_words(what, complete=False)} - use Ctrl+N to add one", timeout=10, + severity="warning") + elif self.selected_comment_guid: self.notify("This comment has no responses yet - use Ctrl+N to add one", timeout=10) else: self.notify("Selected element does not have any comments - use Ctrl+N to add one", timeout=10) diff --git a/my_egeria/my_egeria/ViewSubscriptionsScreen.py b/my_egeria/my_egeria/ViewSubscriptionsScreen.py index 5dd2fdf6..684bdbdc 100644 --- a/my_egeria/my_egeria/ViewSubscriptionsScreen.py +++ b/my_egeria/my_egeria/ViewSubscriptionsScreen.py @@ -12,6 +12,7 @@ from textual.widgets import DataTable, Header, Static, Footer from pyegeria import Egeria, PyegeriaException, load_app_config, settings, print_basic_exception +from paging_support import is_complete, none_words, paged_rows, read_all from profile_utils import element_summary, set_screen_heading @@ -48,39 +49,57 @@ def on_mount(self): set_screen_heading(self, "My Subscriptions") table = self.query_one("#subscriptions_table", DataTable) table.add_columns("Name", "Status", "Description", "GUID") - for row in self.get_my_subscriptions(): + rows = self.get_my_subscriptions() + if rows is None: # the read failed (already said): never "you have none" + return + for row in rows: table.add_row(*row) if table.row_count == 0: - self.notify("You have no digital subscriptions", timeout=5) + if is_complete(rows): + self.notify("You have no digital subscriptions", timeout=5) + else: + self.notify(none_words("digital subscriptions of yours", complete=False), timeout=10, + severity="warning") + elif getattr(rows, "note", None): + self.notify(rows.note, timeout=10, severity="warning") - def get_my_subscriptions(self) -> list[tuple[str, str, str, str]]: - """(name, status, description, GUID) for each DigitalSubscription the user created.""" + def get_my_subscriptions(self) -> list[tuple[str, str, str, str]] | None: + """(name, status, description, GUID) for each DigitalSubscription the user created (PagedRows), or None + if they could not be read.""" eclient = None try: eclient = Egeria(self.view_server, self.platform_url, self.user_name, self.user_password) eclient.create_egeria_bearer_token(self.user_name, self.user_password) - subscriptions = eclient.find_collections( - search_string="*", - metadata_element_type_name="DigitalSubscription", - output_format="JSON", - graph_query_depth=0, # only header/properties are used - ) + # Every subscription is read (paged to Egeria's end), then filtered here by creator + def fetch(start: int, size: int): + return eclient.find_collections( + search_string="*", + metadata_element_type_name="DigitalSubscription", + output_format="JSON", + graph_query_depth=0, # only header/properties are used + start_from=start, + page_size=size, + ) + + read = read_all(fetch, method="find_collections", client=eclient) except PyegeriaException as e: print_basic_exception(e) self.notify(f"Could not retrieve your subscriptions: {e}", timeout=10, severity="error") - return [] + return None finally: if eclient: eclient.close_session() rows = [] - for element in subscriptions if isinstance(subscriptions, list) else []: + for element in read.items: + if not isinstance(element, dict): + continue summary = element_summary(element) if summary.get("created_by") != self.user_name or not summary.get("guid"): continue status = str((element.get("properties") or {}).get("contentStatus") or "") rows.append((summary["name"], status, summary["description"], summary["guid"])) - return rows + return paged_rows(rows, read, "of your digital subscriptions") def action_quit(self): self.dismiss(200) diff --git a/my_egeria/my_egeria/bookmarks_handler.py b/my_egeria/my_egeria/bookmarks_handler.py index e22594fa..532c2614 100644 --- a/my_egeria/my_egeria/bookmarks_handler.py +++ b/my_egeria/my_egeria/bookmarks_handler.py @@ -25,11 +25,18 @@ sys.path.append(str(current_dir)) from pyegeria import Egeria, PyegeriaException +from pyegeria.core._exceptions import PyegeriaNotFoundException +from pyegeria.core._globals import NO_ELEMENTS_FOUND +from pyegeria.core.paging import Exists from MyBookMarksScreen import MyBookMarksScreen +from paging_support import (LookupUnconfirmed, PagedRows, UniqueName, by_unique_name, lookup, paged_rows, read_all, + refusal, unique_refusal) from profile_utils import element_summary, row_identity BOOKMARKS_DISPLAY_NAME = "My Bookmarks" BOOKMARKS_RESOURCE_USE = "Bookmarks" +ROW_HAS_NO_IDENTITY = "That row doesn't identify an Egeria element" + def bookmarks_qualified_name(user_name: str) -> str: @@ -46,15 +53,10 @@ def _bookmarks_client(self) -> Egeria: client.create_egeria_bearer_token(self.user_name, self.user_password) return client - def _find_bookmarks_collection(self, client: Egeria) -> str | None: - """GUID of the user's bookmarks collection, or None if it doesn't exist yet.""" - qualified_name = bookmarks_qualified_name(self.user_name) - response = client.get_collections_by_name(name=qualified_name, output_format="JSON", graph_query_depth=0) - for element in response if isinstance(response, list) else []: - summary = element_summary(element) - if summary.get("qualified_name") == qualified_name and summary.get("guid"): - return summary["guid"] - return None + def _find_bookmarks_collection(self, client: Egeria) -> UniqueName: + """Whether the user's bookmarks collection exists, by Egeria's exact unique-name read of its qualified + name: found (``.guid``), absent (Egeria's specific not-found) or unknown (any other answer or error).""" + return by_unique_name(client, bookmarks_qualified_name(self.user_name), "Collection") def _bookmarks_profile_guid(self, client: Egeria) -> str: """GUID of the user's profile, which owns the bookmarks collection.""" @@ -70,9 +72,15 @@ def _bookmarks_profile_guid(self, client: Egeria) -> str: return guid def _get_or_create_bookmarks_collection(self, client: Egeria) -> str: - guid = self._find_bookmarks_collection(client) - if guid: - return guid + """The bookmarks collection's GUID, created only when Egeria's unique-name read says not-found. + + Raises LookupUnconfirmed (nothing created) on any other answer: creating then could make a second + bookmarks collection.""" + found = self._find_bookmarks_collection(client) + if found.found: + return found.guid + if found.answer != "absent": + raise LookupUnconfirmed(unique_refusal("your bookmarks collection", found)) profile_guid = self._bookmarks_profile_guid(client) body = { "class": "NewElementRequestBody", @@ -98,26 +106,48 @@ def _get_or_create_bookmarks_collection(self, client: Egeria) -> str: # self.log(f"Created bookmarks collection {guid}") return guid - def _bookmark_rows(self, client: Egeria, collection_guid: str) -> list[tuple[str, str, str]]: + def _bookmark_rows(self, client: Egeria, collection_guid: str) -> PagedRows: + """Every member of the bookmarks collection as (name, type, GUID) rows, paged to Egeria's end; + ``complete`` False (and a note) when Egeria didn't confirm there are no more.""" # Depth 0: rows show name/type/guid only (a 30s timeout at the default depth 3 vs ~0.3s). - response = client.get_collection_members(collection_guid=collection_guid, output_format="JSON", - graph_query_depth=0) + def fetch(start: int, size: int) -> Any: + return client.get_collection_members(collection_guid=collection_guid, output_format="JSON", + graph_query_depth=0, start_from=start, page_size=size) + + read = read_all(fetch, method="get_collection_members", client=client) rows = [] - for element in response if isinstance(response, list) else []: + for element in read.items: summary = element_summary(element) if summary.get("guid"): rows.append((summary["name"], summary["type"], summary["guid"])) - return rows + return paged_rows(rows, read, "bookmarks") + + def _is_bookmarked(self, client: Egeria, collection_guid: str, target_guid: str) -> Exists: + """Whether target_guid is already a member of the bookmarks collection (found / absent / unknown).""" + def fetch(start: int, size: int) -> Any: + return client.get_collection_members(collection_guid=collection_guid, output_format="JSON", + graph_query_depth=0, start_from=start, page_size=size) + + return lookup(fetch, lambda element: element_summary(element).get("guid") == target_guid, + method="get_collection_members", client=client) def list_my_bookmarks(self) -> list[tuple[str, str, str]] | None: - """The user's bookmarks as (name, type, GUID) rows, or None if Egeria could not be reached.""" + """The user's bookmarks as (name, type, GUID) rows (PagedRows: ``complete`` says whether Egeria + confirmed that's all of them), or None if Egeria could not be reached.""" client = None try: client = self._bookmarks_client() - collection_guid = self._find_bookmarks_collection(client) - if not collection_guid: - return [] - return self._bookmark_rows(client, collection_guid) + found = self._find_bookmarks_collection(client) + if found.answer == "absent": # Egeria's exact not-found: the user has no bookmarks collection yet + return PagedRows([], complete=True) + if not found.found: # never "no bookmarks": the collection couldn't be read + self.notify(unique_refusal("your bookmarks collection", found, "Nothing is shown"), timeout=10, + severity="error") + return None + rows = self._bookmark_rows(client, found.guid) + if rows.note: + self.notify(rows.note, timeout=10, severity="warning") + return rows except PyegeriaException as e: # self.log(f"Retrieving bookmarks failed: {e}") self.notify(f"Could not retrieve your bookmarks: {e}", timeout=10, severity="error") @@ -144,12 +174,20 @@ def add_my_bookmark(self, target_guid: str) -> bool: try: client = self._bookmarks_client() collection_guid = self._get_or_create_bookmarks_collection(client) - if target_guid in {row[2] for row in self._bookmark_rows(client, collection_guid)}: + already = self._is_bookmarked(client, collection_guid, target_guid) + if already.found: self.notify("That element is already bookmarked", timeout=10, severity="warning") return False + if already.answer != "absent": + self.notify(refusal(f"a bookmark for {target_guid}", already, "Nothing was added"), + timeout=10, severity="warning") + return False client.add_to_collection(collection_guid=collection_guid, element_guid=target_guid) self.notify(f"Bookmarked {target_guid}", timeout=10, severity="information") return True + except LookupUnconfirmed as e: + self.notify(str(e), timeout=10, severity="warning") + return False except (PyegeriaException, ValueError) as e: # self.log(f"Adding bookmark {target_guid} failed: {e}") self.notify(f"Adding the bookmark failed: {e}", timeout=10, severity="error") @@ -158,29 +196,44 @@ def add_my_bookmark(self, target_guid: str) -> bool: if client: client.close_session() - def get_row_guid(self, table: Any, row_key: Any) -> str | None: - """GUID of the element shown in a table row, or None if the row doesn't identify one. + def _resolve_row(self, table: Any, row_key: Any) -> tuple[str | None, str | None]: + """(GUID of the element shown in a table row, None) or (None, why it couldn't be had). - Read from the row's GUID column; rows with only a Qualified Name column - (e.g. Shop for Data's glossaries) are resolved to a GUID through Egeria. - """ + Read from the row's GUID column; rows with only a Qualified Name column (e.g. Shop for Data's + glossaries) are resolved through Egeria's exact unique-name read. A failed read is said as such, + never as "the row doesn't identify an element".""" if table is None or row_key is None: - return None + return None, ROW_HAS_NO_IDENTITY guid, qualified_name = row_identity(table, row_key) - if guid or not qualified_name: - return guid or None + if guid: + return guid, None + if not qualified_name: + return None, ROW_HAS_NO_IDENTITY client = None try: client = self._bookmarks_client() - resolved = client.get_guid_for_name(qualified_name) - # "No elements found" is returned, rather than an exception, when nothing matches - return resolved if isinstance(resolved, str) and resolved and "No " not in resolved else None + resolved = client.get_element_guid_by_unique_name(qualified_name) + except PyegeriaNotFoundException: + return None, f"Egeria holds no element with the qualified name {qualified_name}" except PyegeriaException as e: # self.log(f"Looking up the GUID for {qualified_name} failed: {e}") - return None + return None, f"Couldn't look up the element {qualified_name} in Egeria ({type(e).__name__}); try again" finally: if client: client.close_session() + if resolved == NO_ELEMENTS_FOUND: + return None, f"Egeria holds no element with the qualified name {qualified_name}" + if isinstance(resolved, str) and resolved: + return resolved, None + return None, f"Couldn't look up the element {qualified_name} in Egeria; try again" + + def get_row_guid(self, table: Any, row_key: Any) -> str | None: + """GUID of the element shown in a table row, or None if it couldn't be had (and why is shown when + it was a lookup in Egeria, not the row, that failed).""" + guid, reason = self._resolve_row(table, row_key) + if guid is None and reason and reason != ROW_HAS_NO_IDENTITY: + self.notify(reason, timeout=10, severity="warning") + return guid def bookmark_table_row(self, table: Any, row_key: Any) -> bool: """Bookmark the element shown in a table row, found by its GUID column. @@ -188,10 +241,9 @@ def bookmark_table_row(self, table: Any, row_key: Any) -> bool: Rows with only a Qualified Name column (e.g. Shop for Data's glossaries) are resolved to a GUID first. Returns True if the bookmark was added. """ - guid = self.get_row_guid(table, row_key) + guid, reason = self._resolve_row(table, row_key) if not guid: - self.notify("That row doesn't identify an Egeria element, so it can't be bookmarked", - timeout=10, severity="warning") + self.notify(f"{reason}, so it can't be bookmarked", timeout=10, severity="warning") return False return self.add_my_bookmark(guid) @@ -200,10 +252,15 @@ def delete_my_bookmark(self, target_guid: str) -> bool: client = None try: client = self._bookmarks_client() - collection_guid = self._find_bookmarks_collection(client) - if not collection_guid: + found = self._find_bookmarks_collection(client) + if found.answer == "absent": self.notify("You have no bookmarks to remove", timeout=10, severity="warning") return False + if not found.found: + self.notify(unique_refusal("your bookmarks collection", found, "Nothing was removed"), timeout=10, + severity="warning") + return False + collection_guid = found.guid client.remove_from_collection(collection_guid=collection_guid, element_guid=target_guid) self.notify(f"Removed bookmark {target_guid}", timeout=10, severity="information") return True diff --git a/my_egeria/my_egeria/community_handler.py b/my_egeria/my_egeria/community_handler.py index 7368af37..14845457 100644 --- a/my_egeria/my_egeria/community_handler.py +++ b/my_egeria/my_egeria/community_handler.py @@ -37,6 +37,8 @@ sys.path.append(str(current_dir)) from pyegeria import Egeria, PyegeriaException +from paging_support import (LookupUnconfirmed, PagedRows, by_unique_name, combine, paged_rows, read_all, + unique_refusal) from CommunitiesScreen import CommunitiesScreen, DIRECT_LINK, JOIN_ASSIGNMENT_TYPES, CREATOR_ASSIGNMENT_TYPES from CommunityScreen import comment_type_label @@ -156,8 +158,12 @@ def search_communities(self, search_string: str) -> list[tuple[str, str, str, st try: client = self._community_client() # Depth 0: only the communities' own properties are listed; members are fetched on request - found = client.find_communities(search_string=search_string or "*", ignore_case=True, - output_format="JSON", graph_query_depth=0) + def fetch(start: int, size: int): + return client.find_communities(search_string=search_string or "*", ignore_case=True, + output_format="JSON", graph_query_depth=0, + start_from=start, page_size=size) + + read = read_all(fetch, method="find_communities", client=client) except PyegeriaException as e: # self.log(f"Searching for communities matching {search_string} failed: {e}") self.notify(f"Searching for communities failed: {e}", timeout=10, severity="error") @@ -167,15 +173,15 @@ def search_communities(self, search_string: str) -> list[tuple[str, str, str, st client.close_session() # self.log(f"Communities found for {search_string}: {found}") rows = [] - # "No elements found" is returned, rather than an exception, when nothing matches - for community in found if isinstance(found, list) else []: - guid = _guid(community) + for community in read.items: + guid = _guid(community) if isinstance(community, dict) else "" if not guid: continue properties = community.get("properties") or {} rows.append((_name(properties), properties.get("description") or "", properties.get("qualifiedName") or "", guid)) - return rows + # PagedRows: a note ("at least") when the read stopped short of Egeria's end; "none" only if confirmed + return paged_rows(rows, read, "communities") def get_community_members(self, community_guid: str) -> list[tuple[str, str, str, str, str]] | None: """The people in a community as (name, assignment type, member through, qualified name, GUID) rows, @@ -244,12 +250,20 @@ def get_community_blog_posts(self, community_guid: str) -> list[tuple[str, str, posts = [] try: client = self._community_client() - logs = client.get_attached_note_logs(community_guid, graph_query_depth=0) - # "No elements found" is returned, rather than an exception, when there are none - for log in logs if isinstance(logs, list) else []: - entries = client.get_notes_for_note_log(_guid(log), metadata_element_type_name="BlogEntry", - graph_query_depth=0) - posts += entries if isinstance(entries, list) else [] + logs = self._read_note_logs(client, community_guid) + reads = [logs] + for log in logs.items: + log_guid = _guid(log) if isinstance(log, dict) else "" + if not log_guid: + continue + + def fetch(start: int, size: int, log_guid=log_guid): + return client.get_notes_for_note_log(log_guid, metadata_element_type_name="BlogEntry", + graph_query_depth=0, start_from=start, page_size=size) + + entries = read_all(fetch, method="get_notes_for_note_log", client=client) + reads.append(entries) + posts += [entry for entry in entries.items if isinstance(entry, dict)] except PyegeriaException as e: # self.log(f"Retrieving the blog of community {community_guid} failed: {e}") self.notify(f"Could not retrieve the community's blog: {e}", timeout=10, severity="error") @@ -264,15 +278,34 @@ def get_community_blog_posts(self, community_guid: str) -> list[tuple[str, str, rows.append((create_time, (_name(properties), author, _posted(create_time), properties.get("description") or "", _guid(post)))) rows.sort(key=lambda r: r[0], reverse=True) - return [row for _, row in rows[:LATEST_POSTS]] + latest = [row for _, row in rows[:LATEST_POSTS]] + if combine(*reads): # every read reached a confirmed end: the newest shown are the newest there are + return PagedRows(latest, complete=True) + # Some posts may not have been read, so the newest shown may not be the newest there is + note = (f"Showing the newest of at least {len(rows)} posts read; Egeria didn't confirm there are no more, " + f"so newer posts may be missing") + self.notify(note, timeout=10, severity="warning") + return PagedRows(latest, complete=False, note=note) + + def _read_note_logs(self, client: Egeria, community_guid: str): + """Every note log attached to the community, paged to Egeria's end (a PageResult).""" + def fetch(start: int, size: int): + return client.get_attached_note_logs(community_guid, graph_query_depth=0, start_from=start, + page_size=size) + + return read_all(fetch, method="get_attached_note_logs", client=client) def _community_blog_log(self, client: Egeria, community_guid: str, community_name: str) -> str: - """The GUID of the community's blog note log, which is created on first use.""" + """The GUID of the community's blog note log, which is created on first use: only when Egeria's exact + unique-name read of its qualified name says not-found. Raises LookupUnconfirmed (nothing created) on any + other answer, since creating then could give the community a second blog.""" qualified_name = BLOG_LOG_QN.format(community_guid=community_guid) - logs = client.get_attached_note_logs(community_guid, graph_query_depth=0) - for log in logs if isinstance(logs, list) else []: - if (log.get("properties") or {}).get("qualifiedName") == qualified_name: - return _guid(log) + found = by_unique_name(client, qualified_name, "NoteLog") + if found.found: + return found.guid + if found.answer != "absent": + raise LookupUnconfirmed(unique_refusal(f"a blog for {community_name}", found, + "No blog was created and nothing was posted")) body = { "class": "NewAttachmentRequestBody", "properties": { @@ -305,6 +338,9 @@ def add_community_blog_post(self, community_guid: str, community_name: str, titl self.notify(f"Your post '{title}' has been added to the {community_name} blog", timeout=10, severity="information") return True + except LookupUnconfirmed as e: + self.notify(str(e), timeout=10, severity="warning") + return False except PyegeriaException as e: # self.log(f"Posting to the blog of community {community_guid} failed: {e}") self.notify(f"Posting to the community's blog failed: {e}", timeout=10, severity="error") @@ -319,7 +355,12 @@ def get_community_comments(self, community_guid: str) -> list[tuple[str, str, st client = None try: client = self._community_client() - comments = client.get_attached_comments(community_guid, graph_query_depth=0) + + def fetch(start: int, size: int): + return client.get_attached_comments(community_guid, graph_query_depth=0, start_from=start, + page_size=size) + + read = read_all(fetch, method="get_attached_comments", client=client) except PyegeriaException as e: # self.log(f"Retrieving the comments on community {community_guid} failed: {e}") self.notify(f"Could not retrieve the community's comments: {e}", timeout=10, severity="error") @@ -328,14 +369,19 @@ def get_community_comments(self, community_guid: str) -> list[tuple[str, str, st if client: client.close_session() rows = [] - for comment in comments if isinstance(comments, list) else []: + for comment in read.items: + if not isinstance(comment, dict): + continue properties = comment.get("properties") or {} author, create_time = _created(comment) comment_type = comment_type_label(properties.get("commentType") or "") rows.append((create_time, (properties.get("description") or "", comment_type, author, _posted(create_time), _guid(comment)))) rows.sort(key=lambda r: r[0], reverse=True) - return [row for _, row in rows] + result = paged_rows([row for _, row in rows], read, "comments") + if result.note: + self.notify(result.note, timeout=10, severity="warning") + return result def add_community_comment(self, community_guid: str, text: str, comment_type: str) -> bool: """Attach a comment to the community. comment_type is one of pyegeria's COMMENT_TYPES. @@ -372,13 +418,17 @@ def create_community(self, name: str, description: str, mission: str = "", try: client = self._community_client() qualified_name = client.__create_qualified_name__(COMMUNITY_TYPE, name) - # Egeria rejects a second element with the same qualified name, but only with a server error - existing = client.get_communities_by_name(filter_string=qualified_name, graph_query_depth=0, - output_format="JSON") - if isinstance(existing, list) and existing: + + # Egeria rejects a second element with the same qualified name, but only with a server error, so + # ask its exact unique-name read first: create only on its specific not-found + existing = by_unique_name(client, qualified_name) + if existing.found: self.notify(f"A community called {name} already exists - find it below to join it", timeout=10, severity="warning") return None + if existing.answer != "absent": + self.notify(unique_refusal(f"a community called {name}", existing), timeout=10, severity="warning") + return None properties["qualifiedName"] = qualified_name community_guid = client.create_community(body={"class": "NewElementRequestBody", "isOwnAnchor": True, "properties": properties}) diff --git a/my_egeria/my_egeria/feedback_handler.py b/my_egeria/my_egeria/feedback_handler.py index de8da89e..f7ab11d2 100644 --- a/my_egeria/my_egeria/feedback_handler.py +++ b/my_egeria/my_egeria/feedback_handler.py @@ -28,6 +28,8 @@ sys.path.append(str(current_dir)) from pyegeria import Egeria, PyegeriaException +from paging_support import (LookupUnconfirmed, PagedRows, async_by_unique_name, async_read_all, paged_rows, + unique_refusal) from FeedbackScreens import FeedbackScreen, FeedbackLogScreen FEEDBACK_LOG_OWNER = "garygeeke" @@ -61,7 +63,11 @@ async def feedback_callback(self, feedback: Any) -> None: client = None try: client = self._feedback_client() - log_guid = await self._get_or_create_feedback_log(client) + try: + log_guid = await self._get_or_create_feedback_log(client) + except LookupUnconfirmed as e: + self.notify(f"{e} Your feedback was not saved.", timeout=10, severity="warning") + return if not log_guid: self.notify("Unable to locate or create the feedback log, feedback not saved", timeout=10, severity="error") @@ -93,22 +99,25 @@ def _build_feedback_note(self, log_guid: str, feedback: dict) -> dict: }, } - async def _find_feedback_log(self, client: Egeria) -> str | None: - response = await client._async_get_note_logs_by_name(FEEDBACK_LOG_QN, graph_query_depth=0, - output_format="JSON") - if isinstance(response, list) and response: - return response[0].get("elementHeader", {}).get("guid") - return None + async def _find_feedback_log(self, client: Egeria): + """Whether the feedback log exists, by Egeria's exact unique-name read of FEEDBACK_LOG_QN (a + paging_support.UniqueName: found / absent (Egeria's specific not-found) / unknown).""" + return await async_by_unique_name(client, FEEDBACK_LOG_QN, "NoteLog") async def _get_or_create_feedback_log(self, client: Egeria) -> str | None: """Return the feedback log's GUID, creating the log on first use. When the owner creates it, it is attached to their profile. If another user gets there first the log is created standalone - it is still found by name. + + The log is created only when Egeria's unique-name read says not-found; on any other answer + LookupUnconfirmed is raised and nothing is created (a second feedback log would split the feedback). """ - log_guid = await self._find_feedback_log(client) - if log_guid: - return log_guid + found = await self._find_feedback_log(client) + if found.found: + return found.guid + if found.answer != "absent": + raise LookupUnconfirmed(unique_refusal("the feedback log", found)) properties = { "class": "NoteLogProperties", "typeName": "NoteLog", @@ -143,19 +152,30 @@ async def action_view_feedback_log(self) -> None: finally: if client: client.close_session() + if getattr(entries, "note", None): + self.notify(entries.note, timeout=10, severity="warning") await self.push_screen(FeedbackLogScreen(entries)) async def get_feedback_entries(self, client: Egeria) -> list[dict]: - """Retrieve the notes in the feedback log, newest first.""" - log_guid = await self._find_feedback_log(client) - if not log_guid: - return [] - notes = await client._async_get_notes_for_note_log(log_guid, metadata_element_type_name="Note", - graph_query_depth=0, output_format="JSON") - if not isinstance(notes, list): - return [] + """Retrieve the notes in the feedback log, newest first (PagedRows: ``complete`` False when Egeria + didn't confirm that's all of them).""" + found = await self._find_feedback_log(client) + if found.answer == "absent": # Egeria's exact not-found: no feedback has been recorded yet + return PagedRows([], complete=True) + if not found.found: + return PagedRows([], complete=False, note=unique_refusal("the feedback log", found, "Nothing is shown")) + log_guid = found.guid + + async def fetch(start: int, size: int): + return await client._async_get_notes_for_note_log(log_guid, metadata_element_type_name="Note", + graph_query_depth=0, output_format="JSON", + start_from=start, page_size=size) + + read = await async_read_all(fetch, method="_async_get_notes_for_note_log", client=client) entries = [] - for note in notes: + for note in read.items: + if not isinstance(note, dict): + continue props = note.get("properties") or {} extra = props.get("additionalProperties") or {} versions = (note.get("elementHeader") or {}).get("versions") or {} @@ -167,4 +187,4 @@ async def get_feedback_entries(self, client: Egeria) -> list[dict]: "text": props.get("description", ""), }) entries.sort(key=lambda e: str(e["date"]), reverse=True) - return entries + return paged_rows(entries, read, "feedback entries") diff --git a/my_egeria/my_egeria/my_profile.py b/my_egeria/my_egeria/my_profile.py index 247fbf68..cb8d5957 100644 --- a/my_egeria/my_egeria/my_profile.py +++ b/my_egeria/my_egeria/my_profile.py @@ -93,6 +93,7 @@ from elements_crud_handler import ElementsCrudMixin from feedback_handler import FeedbackMixin, FEEDBACK_LOG_OWNER from bookmarks_handler import BookmarksMixin, bookmarks_qualified_name +from paging_support import paged_rows, read_all, report_empty_cells from MyBookMarksScreen import MyBookMarksScreen from network_handler import NetworkMixin from community_handler import CommunitiesMixin @@ -471,7 +472,13 @@ def _load_my_collections(self) -> list[dict[str, str]]: user_id=self.user_name, user_pwd=self.user_password) eclient.create_egeria_bearer_token(self.user_name, self.user_password) # Depth 0: only header/properties are summarised (~70s at the default depth 3 vs <1s). - response = eclient.find_collections(search_string="*", output_format="JSON", graph_query_depth=0) + # Every collection is read (paged to Egeria's end), then filtered here by creator: one page of + # all collections was shown as the whole list before. + def fetch(start: int, size: int): + return eclient.find_collections(search_string="*", output_format="JSON", graph_query_depth=0, + start_from=start, page_size=size) + + read = read_all(fetch, method="find_collections", client=eclient) except PyegeriaException as e: # self.log(f"Error retrieving My Collections: {e!s}") return [] @@ -479,9 +486,10 @@ def _load_my_collections(self) -> list[dict[str, str]]: if eclient: eclient.close_session() bookmarks_qn = bookmarks_qualified_name(self.user_name) - return [summary for summary in (element_summary(e) for e in (response if isinstance(response, list) else [])) + mine = [summary for summary in (element_summary(e) for e in read.items) if summary.get("created_by") == self.user_name and summary.get("guid") and summary.get("qualified_name") != bookmarks_qn] + return paged_rows(mine, read, "of your collections") def new_profile_return(self, result: int) -> None: """This function handles either the return from the create new profile screen or @@ -607,6 +615,8 @@ def _populate_tables_sync(self) -> Any: for collection in getattr(self, "my_collections", []): self.my_collections_table.add_row( collection["name"], collection["description"], collection["guid"]) + if getattr(getattr(self, "my_collections", None), "note", None): + self.notify(self.my_collections.note, timeout=10, severity="warning") # Populate rows if self.projects_table: @@ -812,10 +822,10 @@ def get_data_product_catalog_table(self) -> int: # self.log(f"Digital Product Catalog data extracted: {self.digital_product_catalog_data_extract}") if not self.digital_product_catalog_data_extract: # self.log(f"No digital product catalog data found for user: {self.user_name}") - self.digital_product_catalog_table.add_row("No digital product catalogs found", - "No data returned from Egeria", "") + headline, detail = report_empty_cells("digital product catalogs", self.digital_product_catalog_data) + self.digital_product_catalog_table.add_row(headline, detail, "") if hasattr(self, "notify"): - self.notify(f"No digital product catalogs found for user: {self.user_name}") + self.notify(f"{headline} for user: {self.user_name}") else: for catalog_item in self.digital_product_catalog_data_extract: self.digital_product_catalog_table.add_row( diff --git a/my_egeria/my_egeria/network_handler.py b/my_egeria/my_egeria/network_handler.py index b7d4ad15..20107b07 100644 --- a/my_egeria/my_egeria/network_handler.py +++ b/my_egeria/my_egeria/network_handler.py @@ -23,9 +23,17 @@ sys.path.append(str(current_dir)) from pyegeria import Egeria, PyegeriaException +from paging_support import element_guid, lookup, paged_rows, read_all, refusal from MyNetworkScreen import MyNetworkScreen PEER_RELATIONSHIP = "Peer" +PEERS_BODY = { + "class": "ResultsRequestBody", + "metadataElementTypeName": "Referenceable", + "includeOnlyRelationships": ["ContactThrough"], +} +#: Rows per people-search request (the search used to read one page of 10 and show it as the whole answer) +PEOPLE_PAGE = 25 class NetworkMixin: @@ -43,14 +51,7 @@ def list_my_peers(self) -> list[tuple[str, str, str, str, str, str]] | None: client = None try: client = self._network_client() - body = { - "class": "ResultsRequestBody", - "metadataElementTypeName": "Referenceable", - "includeOnlyRelationships": ["ContactThrough"], - } - peers = client.get_related_elements(element_guid=self.user_GUID, - relationship_type=PEER_RELATIONSHIP, - body=body) + read = read_all(self._peers_fetch(client), method="get_related_elements", client=client) except PyegeriaException as e: # self.log(f"Retrieving peers failed: {e}") self.notify(f"Could not retrieve your network: {e}", timeout=10, severity="error") @@ -60,8 +61,9 @@ def list_my_peers(self) -> list[tuple[str, str, str, str, str, str]] | None: client.close_session() # self.log(f"Peers: {peers}") rows = [] - # "No elements found" is returned, rather than an exception, when the user has no peers - for peer in peers if isinstance(peers, list) else []: + for peer in read.items: + if not isinstance(peer, dict): + continue properties = peer.get("properties") or {} if properties.get("class") != "PersonProperties": continue @@ -76,7 +78,17 @@ def list_my_peers(self) -> list[tuple[str, str, str, str, str, str]] | None: properties.get("pronouns") or "", guid, relationship_properties.get("description") or "")) - return rows + result = paged_rows(rows, read, "people in your network") + if result.note: + self.notify(result.note, timeout=10, severity="warning") + return result + + def _peers_fetch(self, client: Egeria): + """fetch(start, size) for the user's Peer-related elements (paging goes into the body: 6.2.36+).""" + def fetch(start: int, size: int): + return client.get_related_elements(element_guid=self.user_GUID, relationship_type=PEER_RELATIONSHIP, + body=dict(PEERS_BODY), start_from=start, page_size=size) + return fetch def show_my_network(self) -> None: """Show the My Network screen for the current user.""" @@ -99,9 +111,14 @@ def search_people(self, search_string: str) -> list[tuple[str, str, str]] | None client = None try: client = self._network_client() - found = client.find_actor_profiles(search_string=search_string, ignore_case=True, - output_format="DICT", report_spec="User-Identities", - query_depth=0, page_size=10) + + def fetch(start: int, size: int): + page = client.find_actor_profiles(search_string=search_string, ignore_case=True, + output_format="DICT", report_spec="User-Identities", + query_depth=0, start_from=start, page_size=size) + return [page] if isinstance(page, dict) else page # one match comes back as a dict + + read = read_all(fetch, method="find_actor_profiles", client=client, fetch_page_size=PEOPLE_PAGE) except PyegeriaException as e: # self.log(f"Searching for {search_string} failed: {e}") self.notify(f"Searching for people failed: {e}", timeout=10, severity="error") @@ -110,15 +127,15 @@ def search_people(self, search_string: str) -> list[tuple[str, str, str]] | None if client: client.close_session() # self.log(f"People found for {search_string}: {found}") - if isinstance(found, dict): - found = [found] rows = [] - for person in found if isinstance(found, list) else []: + for person in read.items: + if not isinstance(person, dict): + continue guid = person.get("GUID") # the current user can't be their own peer if guid and guid != self.user_GUID: rows.append((person.get("Display Name") or "", guid, person.get("Qualified Name") or "")) - return rows + return paged_rows(rows, read, "people") def link_peer(self, peer_guid: str, nickname: str = "", description: str = "") -> bool: """Add the person with the given GUID to the user's network. Returns True on success.""" @@ -135,6 +152,17 @@ def link_peer(self, peer_guid: str, nickname: str = "", description: str = "") - client = None try: client = self._network_client() + # The search hides people already in the network only as far as the network list was read, so check + # this person isn't already a peer: a second Peer link would duplicate them + already = lookup(self._peers_fetch(client), lambda peer: element_guid(peer) == peer_guid, + method="get_related_elements", client=client) + if already.found: + self.notify("That person is already in your network", timeout=10, severity="warning") + return False + if already.answer != "absent": + self.notify(refusal("a link to that person in your network", already, "Nothing was added"), + timeout=10, severity="warning") + return False client.link_peer_person(self.user_GUID, peer_guid, body) self.notify("Added to your network", timeout=10, severity="information") return True diff --git a/my_egeria/my_egeria/paging_support.py b/my_egeria/my_egeria/paging_support.py new file mode 100644 index 00000000..85a888c6 --- /dev/null +++ b/my_egeria/my_egeria/paging_support.py @@ -0,0 +1,240 @@ +""" + PDX-License-Identifier: Apache-2.0 + Copyright Contributors to the ODPi Egeria project. + + Egeria's paging contract, as My Profile's handlers use it. + + A list from Egeria is one page of matches; an empty list means that page was filtered empty (keep + paging); only null means there are no more, and on some paths not even that. So a handler never treats + one page as the whole set, and never says "none" or a total unless the read reached a confirmed end. + + - ``read_all`` pages a list call through pyegeria's pager and returns ``PagedRows``: a plain list (so + callers and screens are unchanged) that also says whether the read was complete, and the + "at least N" words to show when it wasn't. + - ``by_unique_name`` / ``async_by_unique_name``: the write guards keyed on a qualified name ask Egeria's + exact unique-name read: found / absent (Egeria's specific not-found) / unknown (anything else: refuse + with ``unique_refusal`` and write nothing). + - ``lookup`` / ``async_lookup`` answer "does Egeria already hold X?" three ways by paging (``exists``) + for the guards keyed on a GUID (collection membership, a Peer link): create only on absent; on unknown, + refuse (``refusal``) and write nothing. + - ``none_words`` is the empty-state text: "No X" only after a confirmed end. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Awaitable, Callable, Iterable + +from pyegeria.core._exceptions import PyegeriaNotFoundException +from pyegeria.core._globals import NO_ELEMENTS_FOUND + +from pyegeria.core.paging import (PageResult, Exists, async_exists, async_page_all, async_paging_for, at_least_note, + exists, page_all, paging_for) + +#: Requests per read for a display list: 20 pages of 100 rows. Past it the list says "at least". +DISPLAY_MAX_PAGES = 20 +NOT_CONFIRMED = "Egeria didn't confirm there are no more" + + +class LookupUnconfirmed(ValueError): + """A write guard couldn't tell whether the thing it would create already exists; nothing was written. + The message is the ``refusal`` words.""" + + +class PagedRows(list): + """The rows a handler shows, plus how the read that produced them ended. + + - ``complete``: Egeria *confirmed* there are no more (a trusted path, or a gated one on a #9453 + platform). Only then is the list shown plainly: a plain count, and "none" for an empty list. + Otherwise ``note`` holds the "Showing at least N ..." words, and an empty list says "none found in + what Egeria returned" (``none_words``). This is the display rule (pager ruling (A), condition 2). + - ``reached_end``: the read got to Egeria's end (a trusted null, or a null then empty probes). That is + the bar for a *write* acting on the end, never for display words; kept for the trace.""" + + def __init__(self, rows: Iterable[Any] = (), *, complete: bool = True, reached_end: bool | None = None, + note: str | None = None): + super().__init__(rows) + self.complete = complete + self.reached_end = complete if reached_end is None else (reached_end or complete) + self.note = note + + +def paged_rows(rows: Iterable[Any], read: PageResult | None, what: str) -> PagedRows: + """``rows`` (built from ``read.items``) as PagedRows: the "at least" note when the read didn't reach + Egeria's end. ``read`` None means the rows needed no read (complete).""" + rows = list(rows) + if read is None or read.complete: + return PagedRows(rows, complete=True) + # display words key on a confirmed end: "at least" on every other end, null_then_empty_probes included + return PagedRows(rows, complete=False, reached_end=read.reached_end, + note=at_least_note(read, what, count=len(rows))) + + +def is_complete(rows: Any) -> bool: + """Whether Egeria confirmed a handler's rows are the whole set, so an empty one may say "none" (a plain + list from an unpaged source counts as confirmed).""" + return getattr(rows, "complete", True) + + +def combine(*reads: Any) -> bool: + """True only when every read (PageResult or PagedRows) reached a confirmed end.""" + return all(getattr(r, "complete", True) for r in reads) + + +def count_text(rows: Any) -> str: + """The count of a handler's rows as it may be shown: "N" on a confirmed end, else "at least N".""" + return str(len(rows)) if is_complete(rows) else f"at least {len(rows)}" + + +def none_words(what: str, complete: bool, matching: str | None = None) -> str: + """The empty-state text: "No " only after a confirmed end, else "none found in what Egeria + returned".""" + suffix = f" matching {matching}" if matching else "" + if complete: + return f"No {what} found{suffix}" + return f"No {what} found{suffix} in what Egeria returned; {NOT_CONFIRMED}" + + +def report_end_complete(result: Any) -> bool: + """Whether an ``exec_report_spec`` result reached a confirmed end. Its ``end`` is "confirmed" only then; + an older executor (no ``end``) counts as unconfirmed for an empty result.""" + return isinstance(result, dict) and result.get("end") == "confirmed" + + +def report_empty_words(what: str, result: Any, matching: str | None = None) -> str: + """Empty-state text for an ``exec_report_spec`` result that has no rows.""" + return none_words(what, report_end_complete(result), matching) + + +def report_empty_cells(what: str, result: Any) -> tuple[str, str]: + """(headline, detail) for the placeholder row of a table whose ``exec_report_spec`` read showed no rows. + + Three different things, said differently: the read failed (``result`` isn't the executor's dict: the + handlers store a sentinel list or an error string then), Egeria confirmed there are none, or the read + stopped without that confirmation.""" + if not isinstance(result, dict): + detail = result if isinstance(result, str) and result else "the read failed; see the log" + return f"Couldn't retrieve {what} from Egeria", detail + if report_end_complete(result): + return f"No {what} found", "Egeria confirmed there are none" + notes = result.get("notes") or [] + detail = "; ".join(str(n) for n in notes) if notes else NOT_CONFIRMED + return f"No {what} found in what Egeria returned", detail + + +def read_all(fetch: Callable[[int, int], Any], *, method: str, client: Any = None, + max_pages: int = DISPLAY_MAX_PAGES, **kwargs) -> PageResult: + """Every row ``fetch(start, size)`` returns, paged per Egeria's contract for ``method``.""" + return page_all(fetch, max_pages=max_pages, **kwargs, **paging_for(method, client)) + + +async def async_read_all(fetch: Callable[[int, int], Awaitable[Any]], *, method: str, client: Any = None, + max_pages: int = DISPLAY_MAX_PAGES, **kwargs) -> PageResult: + """``read_all`` for an async ``fetch``.""" + return await async_page_all(fetch, max_pages=max_pages, **kwargs, **(await async_paging_for(method, client))) + + +def lookup(fetch: Callable[[int, int], Any], match: Callable[[Any], bool], *, method: str, + client: Any = None, **kwargs) -> Exists: + """found / absent / unknown: whether a row satisfying ``match`` (exact, client-side) is in what + ``fetch`` pages to. Use only where one match decides the outcome.""" + return exists(fetch, match, method=method, client=client, **kwargs) + + +async def async_lookup(fetch: Callable[[int, int], Awaitable[Any]], match: Callable[[Any], bool], *, method: str, + client: Any = None, **kwargs) -> Exists: + """``lookup`` for an async ``fetch``.""" + return await async_exists(fetch, match, method=method, client=client, **kwargs) + + +def refusal(what: str, found: Exists, action: str = "Nothing was created") -> str: + """The words a write guard shows when it can't tell whether ``what`` already exists.""" + read = found.read + return (f"Couldn't confirm whether Egeria already holds {what}: it didn't confirm there are no more " + f"({read.positions_read} positions read, {read.end_reason or read.end}). {action}; try again later.") + + +# ── Guards keyed on a qualified name: Egeria's unique-name read (read-back R2) ───────────────────────────── +# A qualified name is answered by Egeria's own exact, server-side read (getMetadataElementByUniqueName), with +# no client paging, so no B5 and no filtered page can make it read as "absent" (the publish ruling): +# found -> reuse; Egeria's specific not-found (NO_ELEMENTS_FOUND or PyegeriaNotFoundException) -> absent -> +# create; any other answer or error -> unknown -> refuse, write nothing. + +@dataclass(frozen=True) +class UniqueName: + """Egeria's answer to "which element has this qualified name?": found / absent / unknown.""" + answer: str + qualified_name: str + element: dict | None = None + reason: str | None = None + + @property + def found(self) -> bool: + return self.answer == "found" + + @property + def guid(self) -> str: + return element_guid(self.element) + + +def _type_names(element: dict) -> list[str]: + t = (element.get("elementHeader") or {}).get("type") or {} + return [n for n in (t.get("typeName"), *(t.get("superTypeNames") or [])) if n] + + +def _unique_answer(result: Any, qualified_name: str, type_name: str | None) -> UniqueName: + if result == NO_ELEMENTS_FOUND: + return UniqueName("absent", qualified_name) + element = result if isinstance(result, dict) and result.get("elementHeader") else ( + result.get("element") if isinstance(result, dict) and isinstance(result.get("element"), dict) else None) + if element is None or not element_guid(element): + return UniqueName("unknown", qualified_name, reason="Egeria's answer wasn't an element or 'not found'") + if type_name and type_name not in _type_names(element): + held = (_type_names(element) or ["different element"])[0] + return UniqueName("unknown", qualified_name, element, + reason=f"Egeria holds a {held} with that qualified name, not a {type_name}") + return UniqueName("found", qualified_name, element) + + +def _unique_failed(exc: BaseException, qualified_name: str) -> UniqueName: + if isinstance(exc, PyegeriaNotFoundException): + return UniqueName("absent", qualified_name) + return UniqueName("unknown", qualified_name, reason=f"the unique-name read failed: {type(exc).__name__}") + + +def by_unique_name(client: Any, qualified_name: str, type_name: str | None = None) -> UniqueName: + """Egeria's exact unique-name read of ``qualified_name`` (optionally checking the element's type).""" + try: + result = client.get_element_by_unique_name(qualified_name) + except Exception as exc: # noqa: BLE001 -- only Egeria's specific not-found is "absent" + return _unique_failed(exc, qualified_name) + return _unique_answer(result, qualified_name, type_name) + + +async def async_by_unique_name(client: Any, qualified_name: str, type_name: str | None = None) -> UniqueName: + """``by_unique_name`` for async code.""" + try: + result = await client._async_get_element_by_unique_name(qualified_name) + except Exception as exc: # noqa: BLE001 -- only Egeria's specific not-found is "absent" + return _unique_failed(exc, qualified_name) + return _unique_answer(result, qualified_name, type_name) + + +def unique_refusal(what: str, found: UniqueName, action: str = "Nothing was created") -> str: + """The words a QN-keyed write guard shows when Egeria's answer was neither the element nor not-found.""" + return (f"Couldn't confirm whether Egeria already holds {what} ({found.qualified_name}): " + f"{found.reason or 'no clear answer'}. {action}; try again later.") + + +def element_qn(element: Any) -> str: + """An Egeria 6 element's qualifiedName ('' when it has none).""" + if not isinstance(element, dict): + return "" + return str((element.get("properties") or {}).get("qualifiedName") or "") + + +def element_guid(element: Any) -> str: + """An Egeria 6 element's GUID ('' when it has none).""" + if not isinstance(element, dict): + return "" + return str((element.get("elementHeader") or {}).get("guid") or "") diff --git a/my_egeria/my_egeria/shop_for_data_handler.py b/my_egeria/my_egeria/shop_for_data_handler.py index 3b79cd39..bf1469dd 100644 --- a/my_egeria/my_egeria/shop_for_data_handler.py +++ b/my_egeria/my_egeria/shop_for_data_handler.py @@ -42,6 +42,33 @@ from CreateSubscriptionRequestScreen import CreateSubscriptionRequestScreen from StatusScreen import StatusScreen from GenericDataViewScreen import GenericDataViewScreen +from paging_support import NOT_CONFIRMED, lookup, report_empty_cells, report_empty_words + +#: Rows per request when looking for a product's data set. Never 1: on Egeria builds before odpi/egeria #9453 +#: a page of 1 can say "No elements found" though a match exists (the B5 bug), so the old page_size=1 lookup +#: could report a data set as missing. +DATA_SET_PAGE = 25 + + +def find_data_set(client: Any, search_string: str): + """The first tabular data set Egeria returns for search_string (its own starts-with search), as a + paging.Exists: found (``matches[0]`` is the DICT row), absent (the read reached Egeria's end) or unknown. + Paged per Egeria's contract, so an empty or filtered first page is not read as "none".""" + def fetch(start: int, size: int): + page = client.find_tabular_data_sets(search_string=search_string, start_from=start, page_size=size, + graph_query_depth=0, # only the GUID of the first row is used + output_format="DICT") + return [page] if isinstance(page, dict) else page # a single match may come back as a dict + + return lookup(fetch, lambda row: isinstance(row, dict) and bool(row.get("GUID")), + method="find_tabular_data_sets", client=client, fetch_page_size=DATA_SET_PAGE) + + +def data_set_absence_words(found: Any, name: str) -> str: + """What to say when no data set was found for name: plain "none" only when the read reached the end.""" + if found.answer == "absent": + return f"No metadata found for dataset {name}" + return f"No metadata found for dataset {name} in what Egeria returned; {NOT_CONFIRMED}" class ShopForDataMixin(): """Mixin class providing Catalogs & Shop For Data functionality for MyProfileApp.""" @@ -239,7 +266,7 @@ def on_worker_state_changed(self, event: Worker.StateChanged) -> None: if glossary_table is not None: if not self.glossary_data_extract: self.app.log(f"No glossary data found for search string: {self.selected_t_node}") - glossary_table.add_row("No glossaries found", "No data returned from Egeria", "") + glossary_table.add_row(*report_empty_cells("glossaries", self.glossary_data), "") else: for g in self.glossary_data_extract: glossary_table.add_row(g.get("Display Name", ""), g.get("Description", ""), g.get("Qualified Name", "")) @@ -253,8 +280,8 @@ def on_worker_state_changed(self, event: Worker.StateChanged) -> None: if digital_product_catalog_table is not None: if not self.digital_product_catalog_data_extract: self.app.log(f"No digital product catalog data found for user: {self.user_name}") - digital_product_catalog_table.add_row("No digital product catalogs found", - "No data returned from Egeria", "", "") + digital_product_catalog_table.add_row( + *report_empty_cells("digital product catalogs", self.digital_product_catalog_data), "", "") else: for catalog_item in self.digital_product_catalog_data_extract: digital_product_catalog_table.add_row( @@ -273,7 +300,8 @@ def on_worker_state_changed(self, event: Worker.StateChanged) -> None: if data_dictionary_table is not None: if not self.data_dictionary_data_extract: self.app.log(f"No data dictionary details found for user: {self.user_name}") - data_dictionary_table.add_row("No data dictionaries found", "No data returned from Egeria", "", "") + data_dictionary_table.add_row( + *report_empty_cells("data dictionaries", self.data_dictionary_data), "", "") else: self.app.log( f"Found {len(self.data_dictionary_data_extract)} data dictionaries for user {self.user_name}") @@ -294,7 +322,7 @@ def on_worker_state_changed(self, event: Worker.StateChanged) -> None: if business_domain_table is not None: if not self.business_domain_data_extract: self.app.log(f"No business domains found for user {self.user_name}") - business_domain_table.add_row("No business domains found", "No data returned from Egeria", "") + business_domain_table.add_row(*report_empty_cells("business domains", self.business_domain_data), "") else: self.app.log( f"Found {len(self.business_domain_data_extract)} business domains for user {self.user_name}") @@ -313,8 +341,7 @@ def on_worker_state_changed(self, event: Worker.StateChanged) -> None: self.app.log(f"Found {len(root_extract)} root collections for user {self.user_name}") if root_collection_table is not None: if not root_extract: - err_msg = str(self.collections) if isinstance(self.collections, str) else "No data returned from Egeria" - root_collection_table.add_row("No root collections found", err_msg, "") + root_collection_table.add_row(*report_empty_cells("root collections", self.collections), "") else: for collection in root_extract: root_collection_table.add_row( @@ -555,7 +582,7 @@ def build_dictionary_details(self, target_qualified_name: str, target_display_na elif isinstance(self.dictionary_details, dict) and self.dictionary_details.get("kind") == "empty": dictionary_tree: Tree = Tree(label="Empty Dictionary", id="data_dictionary_tree") dictionary_tree.root.expand() - dictionary_tree.root.content = "No dictionary terms found for this dictionary" + dictionary_tree.root.content = report_empty_words("dictionary terms", self.dictionary_details) + " for this dictionary" else: dictionary_tree = Tree(label=self.dictionary_display_name, id="data_dictionary_tree") dictionary_tree.root.expand() @@ -624,7 +651,7 @@ def build_domain_details(self, target_qualified_name: str, target_type__name: st elif isinstance(self.domain_details, dict) and self.domain_details.get("kind") == "empty": domain_tree: Tree = Tree(label="Empty Business Domain", id="business_domain_tree") domain_tree.root.expand() - domain_tree.root.content = "No domain details found for this business domain" + domain_tree.root.content = report_empty_words("domain details", self.domain_details) + " for this business domain" else: domain_items = self._extract_report_data(self.domain_details) self.domain_details_data = domain_items @@ -706,7 +733,7 @@ def build_catalog_details(self, target_qualified_name: str, target_display_name: elif isinstance(self.catalog_details, dict) and self.catalog_details.get("kind") == "empty": catalog_tree: Tree = Tree(label="Empty Catalog", id="digital_product_catalog_tree") catalog_tree.root.expand() - catalog_tree.root.content = "No catalog terms found for this catalog" + catalog_tree.root.content = report_empty_words("catalog terms", self.catalog_details) + " for this catalog" sample_data: list = [] else: catalog_tree = Tree(label=self.catalog_display_name, id="digital_product_catalog_tree") @@ -757,16 +784,10 @@ def build_catalog_details(self, target_qualified_name: str, target_display_name: try: dclient = Egeria(self.view_server, self.platform_url, self.user_name, self.user_password) token = dclient.create_egeria_bearer_token(self.user_name, self.user_password) - data_set_metadata = dclient.find_tabular_data_sets( - search_string=membership_qname, - start_from=0, - page_size=1, - graph_query_depth=0, # only the GUID of the first row is used - output_format="DICT", - ) - self.app.log(f"Dataset metadata retrieved: {data_set_metadata}") - if isinstance(data_set_metadata, list) and len(data_set_metadata) > 0 and data_set_metadata != "No elements found": - data_set_guid = data_set_metadata[0].get("GUID") + data_set = find_data_set(dclient, membership_qname) + self.app.log(f"Dataset lookup for {membership_qname}: {data_set.answer}") + if data_set.found: + data_set_guid = data_set.matches[0].get("GUID") if data_set_guid: data_set_data = dclient.get_tabular_data_set( tabular_data_set_guid=data_set_guid, @@ -779,7 +800,7 @@ def build_catalog_details(self, target_qualified_name: str, target_display_name: else: sample_data.append(f"No GUID found for dataset {membership_qname}") else: - sample_data.append(f"No metadata found for dataset {membership_qname}") + sample_data.append(data_set_absence_words(data_set, membership_qname)) except PyegeriaException as e: self.app.log(f"Error retrieving dataset data: {e}") print_basic_exception(e) @@ -1186,23 +1207,15 @@ async def request_to_sample_data_source( try: dclient = Egeria(self.view_server, self.platform_url, self.user_name, self.user_password) dclient.create_egeria_bearer_token(self.user_name, self.user_password) - data_set_metadata = dclient.find_tabular_data_sets( - search_string=element_qname, - start_from=0, - page_size=1, - graph_query_depth=0, # only the GUID of the first row is used - output_format="DICT", - ) - if (not data_set_metadata or data_set_metadata == "No elements found") and element_name: - data_set_metadata = dclient.find_tabular_data_sets( - search_string=element_name, - start_from=0, - page_size=1, - graph_query_depth=0, # only the GUID of the first row is used - output_format="DICT", - ) - if isinstance(data_set_metadata, list) and len(data_set_metadata) > 0 and data_set_metadata != "No elements found": - data_set_guid = data_set_metadata[0].get("GUID") + data_set = find_data_set(dclient, element_qname) + # Fall back to the display name only when the qualified-name read reached the end without a match + if data_set.answer == "absent" and element_name: + data_set = find_data_set(dclient, element_name) + if not data_set.found: + self.app.notify(data_set_absence_words(data_set, element_name or element_qname), + title="Shop for Data", severity="warning") + if data_set.found: + data_set_guid = data_set.matches[0].get("GUID") if data_set_guid: sample_data = dclient.get_tabular_data_set( tabular_data_set_guid=data_set_guid, diff --git a/my_egeria/my_egeria/team_roles_handler.py b/my_egeria/my_egeria/team_roles_handler.py index 6cbeab23..1b3f40a4 100644 --- a/my_egeria/my_egeria/team_roles_handler.py +++ b/my_egeria/my_egeria/team_roles_handler.py @@ -23,6 +23,7 @@ from pyegeria import exec_report_spec, PyegeriaException, print_basic_exception, load_app_config, settings from MyTeamScreen import MyTeam from StatusScreen import StatusScreen +from paging_support import report_empty_words class TeamRolesMixin: @@ -150,7 +151,7 @@ def find_team_members(self, role_name: str) -> list[Any]: else: # self.log(f"No team members found for role: {role_search_key}") error_category = "Team Members" - error_message = "No team members found" + error_message = report_empty_words("team members", team_member_data) # self.log(f"Error retrieving team members: {error_category}, {error_message}") self.push_screen(StatusScreen(f"{error_category}: {error_message}"), callback=self.status_callback) return [[], None, None, None, None] @@ -236,7 +237,7 @@ def display_glossary_term_details(self, term: str) -> int: elif self.term_details.get("kind") == "empty": # self.log(f"No glossary term details found for qualified name: {self.target_term}") error_category = "Glossary Term Details" - error_message = "No glossary term details found" + error_message = report_empty_words("glossary term details", self.term_details) # self.log(f"Error retrieving glossary term details: {error_category}, {error_message}") self.push_screen(StatusScreen(f"{error_category}: {error_message}"), callback=self.status_callback) return 440 diff --git a/my_egeria/my_egeria/tech_types_handler.py b/my_egeria/my_egeria/tech_types_handler.py index c02ac342..a79bbece 100644 --- a/my_egeria/my_egeria/tech_types_handler.py +++ b/my_egeria/my_egeria/tech_types_handler.py @@ -28,6 +28,7 @@ TechnologyTypeTemplatesScreen, TechnologyTypeProcessesScreen, ) +from paging_support import none_words from profile_utils import clean_structure, bools_to_strings @@ -41,7 +42,8 @@ async def handle_technology_types_option(self) -> None: # self.log(f"Tech Type Response: {self.tech_type_response} | {self.tech_type_list}") if self.tech_type_response == []: # self.log("No technology types found.") - self.notify("No technology types are defined in Egeria", timeout=10, severity="warning") + # An empty list from Egeria is a page filtered empty, not proof that none are defined + self.notify(none_words("technology types", complete=False), timeout=10, severity="warning") return elif len(self.tech_type_response) == 3 and int(self.tech_type_response) >= 400: # self.log("Error fetching technology types.") diff --git a/pyegeria/authoring/__init__.py b/pyegeria/authoring/__init__.py index d1d83557..9e2529ac 100644 --- a/pyegeria/authoring/__init__.py +++ b/pyegeria/authoring/__init__.py @@ -29,20 +29,22 @@ publish_plan, withdraw_plan, ) from pyegeria.authoring.models import ( - AuthoringGates, AuthoringRefused, Change, Diff, Draft, ReportDraft, SheetDraft, SheetPlacement, SpecDraft, + ANSWER_ABSENT, ANSWER_REFUSED, ANSWER_UNKNOWN, ANSWERS, AuthoringGates, AuthoringRefused, Change, Diff, Draft, ReportDraft, SheetDraft, SheetPlacement, SpecDraft, ValidationResult, as_draft, default_gates, ) +from pyegeria.authoring.permissions import async_can_edit, can_edit from pyegeria.authoring.preview import async_preview, preview +from pyegeria.authoring.used_by import VIA_KEYS, async_used_by, spec_label_of, used_by from pyegeria.authoring.publication import AUTHORING_KEYS, STAMP_KEYS from pyegeria.authoring.runner import inprocess_runner from pyegeria.authoring.serialize import has_parent_id, report_parameter_maps, serialize from pyegeria.authoring.validate import async_validate, validate __all__ = [ - "AUTHORING_KEYS", "STAMP_KEYS", "AuthoringGates", "AuthoringRefused", "Change", "Diff", "Draft", "DraftStore", "EgeriaDraftStore", + "ANSWERS", "ANSWER_ABSENT", "ANSWER_REFUSED", "ANSWER_UNKNOWN", "AUTHORING_KEYS", "STAMP_KEYS", "AuthoringGates", "AuthoringRefused", "Change", "Diff", "Draft", "DraftStore", "EgeriaDraftStore", "KNOWN_CONSUMERS", "LocalDraftStore", "ReportDraft", "SheetDraft", "SheetPlacement", "SpecDraft", "TEMPLATES", "ValidationResult", "apply", "as_draft", "async_apply", "async_publish_plan", "async_history", "async_load_held", "async_new_from", "async_preview", - "async_validate", "default_gates", "diff", "draft_store", "has_parent_id", "history", "inprocess_runner", + "async_can_edit", "async_used_by", "async_validate", "can_edit", "default_gates", "diff", "draft_store", "has_parent_id", "history", "inprocess_runner", "load_held", "new_draft", "new_from", "preview", "publish_plan", "report_parameter_maps", "serialize", - "validate", "withdraw_plan", + "used_by", "VIA_KEYS", "spec_label_of", "validate", "withdraw_plan", ] diff --git a/pyegeria/authoring/drafts.py b/pyegeria/authoring/drafts.py index d3d54c71..6f169b3f 100644 --- a/pyegeria/authoring/drafts.py +++ b/pyegeria/authoring/drafts.py @@ -21,7 +21,7 @@ from pyegeria.authoring.publication import stamp_of from pyegeria.authoring.models import ( - KINDS, AuthoringRefused, ReportDraft, SheetDraft, SheetPlacement, SpecDraft, as_draft, draft_key, + ANSWER_ABSENT, KINDS, AuthoringRefused, ReportDraft, SheetDraft, SheetPlacement, SpecDraft, as_draft, draft_key, ) from pyegeria.view._output_format_models import Column, Format, FormatSet @@ -355,31 +355,38 @@ async def async_load(self, user_id: str, kind: str, name: str) -> Optional[AnyDr return None return await async_load_held(kind, name, self.client) - async def async_list(self, user_id: str, kind: Optional[str] = None) -> List[Dict[str, Any]]: - """The author's drafts: every ReportSpec, Report and Dashboard in their private zone, read as them.""" + async def async_list(self, user_id: str, kind: Optional[str] = None, *, + notes: Optional[List[str]] = None) -> List[Dict[str, Any]]: + """The author's drafts: every ReportSpec, Report and Dashboard in their private zone, read as them. + + Paged through ``page_all`` (pager step 4; a suspect path, classification-explorer property search). + When a kind's read isn't confirmed complete, ``notes`` (if given) gets "Showing at least N ..." for it, + so a caller that shows the list never presents it as all of them.""" + from pyegeria.core.paging import async_page_all, async_paging_for, at_least_note, log_end zone = self._author(user_id) out: List[Dict[str, Any]] = [] for k in ([kind] if kind else list(KINDS)): - start = 0 - while True: - page = await self.client.classification_manager._async_find_elements_by_property_value( + async def fetch(start, size, k=k): + return await self.client.classification_manager._async_find_elements_by_property_value( ".*", ["qualifiedName"], metadata_element_type=_ELEMENT_TYPE[k], starts_with=False, - governance_zone_filter=[zone], graph_query_depth=0, start_from=start, page_size=50) - raw = [e for e in page if isinstance(e, dict) and e.get("elementHeader")] if isinstance(page, list) else [] - if not raw: # only an empty page ends the read (a short page isn't the last page) - break - start += len(raw) - for el in (e for e in raw if zone in self._zones(e)): - props = el.get("properties") or {} - versions = (el.get("elementHeader") or {}).get("versions") or {} - name = props.get("displayName") or props.get("qualifiedName") - if k == "sheet" and str(props.get("qualifiedName", "")).startswith("Dashboard::"): - name = props["qualifiedName"].split("::", 1)[1] - out.append({"kind": k, "name": name, "qualified_name": props.get("qualifiedName"), - "guid": el["elementHeader"].get("guid"), - "created": versions.get("createTime"), "updated": versions.get("updateTime"), - "zones": self._zones(el), "marked": _is_marked(k, el), - "published_version": stamp_of(el)}) + governance_zone_filter=[zone], graph_query_depth=0, start_from=start, page_size=size) + paged = await async_page_all(fetch, fetch_page_size=50, + **await async_paging_for("find_elements_by_property_value", self.client)) + log_end("drafts.async_list", paged) + if notes is not None and (words := at_least_note(paged, f"{k} drafts")): + notes.append(words) + raw = [e for e in paged.items if isinstance(e, dict) and e.get("elementHeader")] + for el in (e for e in raw if zone in self._zones(e)): + props = el.get("properties") or {} + versions = (el.get("elementHeader") or {}).get("versions") or {} + name = props.get("displayName") or props.get("qualifiedName") + if k == "sheet" and str(props.get("qualifiedName", "")).startswith("Dashboard::"): + name = props["qualifiedName"].split("::", 1)[1] + out.append({"kind": k, "name": name, "qualified_name": props.get("qualifiedName"), + "guid": el["elementHeader"].get("guid"), + "created": versions.get("createTime"), "updated": versions.get("updateTime"), + "zones": self._zones(el), "marked": _is_marked(k, el), + "published_version": stamp_of(el)}) return out async def async_delete(self, user_id: str, kind: str, name: str) -> bool: @@ -426,9 +433,12 @@ async def async_prune_drafts(self, user_id: str, older_than_days: int = PRUNE_DA from datetime import datetime, timedelta, timezone cutoff = datetime.now(timezone.utc) - timedelta(days=older_than_days) zone = self._author(user_id) - drafts = await self.async_list(user_id) + read_notes: List[str] = [] + drafts = await self.async_list(user_id, notes=read_notes) out: Dict[str, Any] = {"removed": [], "kept": [], "not_dated": [], "not_marked": [], "not_deleted": [], "kept_reasons": {}, "dry_run": dry_run, "older_than_days": older_than_days} + if read_notes: # pager step 4: the list read wasn't confirmed whole, so some old drafts may be unseen + out["read_notes"] = read_notes for d in drafts: key = f"{d['kind']}:{d['name']}" when = _parse_time(d.get("updated")) or _parse_time(d.get("created")) @@ -466,7 +476,8 @@ async def async_promotion_plan(self, user_id: str, kind: str, name: str, *, fold self._author(user_id) draft = await self.async_load(user_id, kind, name) if draft is None: - raise AuthoringRefused(f"You have no {kind} draft named {name!r} in your private zone.") + raise AuthoringRefused(f"You have no {kind} draft named {name!r} in your private zone.", + answer=ANSWER_ABSENT) # The target is resolved as the author (its private zone comes back in `held`, so the plan stamps the # element, then declassifies it). return await async_publish_plan(draft, self.client, folders=folders) @@ -483,8 +494,9 @@ def save(self, user_id: str, draft: AnyDraft) -> str: def load(self, user_id: str, kind: str, name: str) -> Optional[AnyDraft]: return self._sync(self.async_load(user_id, kind, name)) - def list(self, user_id: str, kind: Optional[str] = None) -> List[Dict[str, Any]]: - return self._sync(self.async_list(user_id, kind)) + def list(self, user_id: str, kind: Optional[str] = None, *, + notes: Optional[List[str]] = None) -> List[Dict[str, Any]]: + return self._sync(self.async_list(user_id, kind, notes=notes)) def delete(self, user_id: str, kind: str, name: str) -> bool: return self._sync(self.async_delete(user_id, kind, name)) diff --git a/pyegeria/authoring/lifecycle.py b/pyegeria/authoring/lifecycle.py index 997121fb..0eab6ea2 100644 --- a/pyegeria/authoring/lifecycle.py +++ b/pyegeria/authoring/lifecycle.py @@ -14,7 +14,7 @@ from loguru import logger from pyegeria.authoring.models import ( - AuthoringGates, AuthoringRefused, ReportDraft, SheetDraft, SheetPlacement, SpecDraft, as_draft, default_gates, + ANSWER_ABSENT, ANSWER_UNKNOWN, AuthoringGates, AuthoringRefused, ReportDraft, SheetDraft, SheetPlacement, SpecDraft, as_draft, default_gates, ) from pyegeria.authoring.publication import AUTHORING_KEYS from pyegeria.authoring.serialize import ( @@ -48,18 +48,38 @@ async def _by_qualified_name(client: Any, qn: str) -> Optional[Dict[str, Any]]: async def _report_element(client: Any, label: str) -> Optional[Dict[str, Any]]: - """A Report by qualified name, else by exact display name (one Report must match).""" + """A Report by qualified name (Egeria's unique-name read), else by exact display name, three-way (pager + step 6, the Architect's _report_element ruling, 2026-11-14): + - found -> the Report (two or more by display name are refused: give its qualified name); + - absent -> None, only when the display-name read reached its end (trusted on a #9453 platform; never on + the by-property-value path otherwise, which is NULL_NOT_END); + - unknown -> AuthoringRefused "couldn't confirm whether Egeria holds a Report named X", with the read's + trace: never None, so no caller reads it as "holds no" (or acts on it as absent).""" + from pyegeria.core.paging import async_exists el = await _by_qualified_name(client, label) if el and _type_name(el) == "Report": return el - found = await client.classification_manager._async_get_elements_by_property_value( - label, ["displayName"], metadata_element_type="Report", starts_with=False, ends_with=False, - ignore_case=False, page_size=25, graph_query_depth=0) - matches = [e for e in found if isinstance(e, dict) and (e.get("properties") or {}).get("displayName") == label] \ - if isinstance(found, list) else [] - if len(matches) > 1: - raise AuthoringRefused(f"{len(matches)} Reports are named {label!r}; give its qualified name.") - return matches[0] if matches else None + + async def fetch(start, size): + return await client.classification_manager._async_get_elements_by_property_value( + label, ["displayName"], metadata_element_type="Report", starts_with=False, ends_with=False, + ignore_case=False, start_from=start, page_size=size, graph_query_depth=0) + + # Stops at the first page with a match (found is decisive; it never proves the name unique beyond that + # page, but two on it are refused); trusted on a #9453 platform through the client's /api/about. + found = await async_exists(fetch, lambda e: isinstance(e, dict) + and (e.get("properties") or {}).get("displayName") == label, + method="get_elements_by_property_value", fetch_page_size=25, client=client) + if len(found.matches) > 1: + raise AuthoringRefused(f"{len(found.matches)} Reports are named {label!r}; give its qualified name.") + if found.found: + return found.matches[0] + if found.answer == "absent": + return None # "holds no Report named X" + r = found.read + raise AuthoringRefused(f"Couldn't confirm whether Egeria holds a Report named {label!r} (the display-name read " + f"ended {r.end}, {r.end_reason}, after {r.positions_read} positions: Egeria didn't " + f"confirm there are no more). Give the Report's qualified name.", answer=ANSWER_UNKNOWN) def _type_name(el: Dict[str, Any]) -> str: @@ -175,7 +195,7 @@ async def async_load_held(kind: str, label: str, client: Any) -> Union[SpecDraft from pyegeria.view.egeria_spec_cache import _Skip, _build el = await find_report_spec_element(client, label) if not el: - raise AuthoringRefused(f"Egeria holds no report spec {label!r} that you can see.") + raise AuthoringRefused(f"Egeria holds no report spec {label!r} that you can see.", answer=ANSWER_ABSENT) try: built = _build(el, folders=[], identity=getattr(client, "user_id", "") or "") except _Skip as exc: @@ -184,15 +204,16 @@ async def async_load_held(kind: str, label: str, client: Any) -> Union[SpecDraft qualified_name=built.provenance.get("qualified_name"), base_version=built.provenance.get("header_version"), notes=list(built.notes)) if kind == "report": - el = await _report_element(client, label) + el = await _report_element(client, label) # unknown raises "couldn't confirm" (never "holds no") if not el: - raise AuthoringRefused(f"Egeria holds no Report {label!r} that you can see.") + raise AuthoringRefused(f"Egeria holds no Report named {label!r} that you can see.", answer=ANSWER_ABSENT) return report_draft_from_element(el) if kind == "sheet": from pyegeria.view.dashboards import read_sheet_from_chain sheet = await read_sheet_from_chain(client, label) if sheet is None: - raise AuthoringRefused(f"Egeria holds no dashboard sheet {label!r} (Dashboard::{label}) that you can see.") + raise AuthoringRefused(f"Egeria holds no dashboard sheet {label!r} (Dashboard::{label}) that you can see.", + answer=ANSWER_ABSENT) draft = sheet_draft_from_sheet(sheet) dash = await _by_qualified_name(client, f"Dashboard::{label}") draft.base_version = _version(dash) if dash else None diff --git a/pyegeria/authoring/models.py b/pyegeria/authoring/models.py index d6708856..ef3b7f1f 100644 --- a/pyegeria/authoring/models.py +++ b/pyegeria/authoring/models.py @@ -217,5 +217,23 @@ def summary(self) -> Dict[str, Any]: "breaking": [c.model_dump() for c in self.breaking]} +ANSWER_ABSENT = "absent" # a read that reached a confirmed end: Egeria holds no such artifact for this user +ANSWER_UNKNOWN = "unknown" # whether Egeria holds it couldn't be told (a read that didn't reach a confirmed end) +ANSWER_REFUSED = "refused" # anything else: the request itself can't be carried out +ANSWERS = (ANSWER_ABSENT, ANSWER_UNKNOWN, ANSWER_REFUSED) + + class AuthoringRefused(ValueError): - """The request can't be carried out; the words say why and what to do instead.""" + """The request can't be carried out; the words say why and what to do instead. + + `answer` is the machine-readable kind, so callers branch on it, never on the words: + - "absent": Egeria holds no such artifact that this user can see (said only on a confirmed end); + - "unknown": whether Egeria holds it couldn't be told; never to be read as "none"; + - "refused" (the default): any other refusal (a bad argument, a guard, a failed write). + """ + + def __init__(self, *args: Any, answer: str = ANSWER_REFUSED) -> None: + if answer not in ANSWERS: + raise ValueError(f"AuthoringRefused answer is one of {', '.join(ANSWERS)}, not {answer!r}.") + super().__init__(*args) + self.answer = answer diff --git a/pyegeria/authoring/permissions.py b/pyegeria/authoring/permissions.py new file mode 100644 index 00000000..7afa373a --- /dev/null +++ b/pyegeria/authoring/permissions.py @@ -0,0 +1,118 @@ +"""Can this user edit this spec, Report or sheet? (P2, for "Edit in Studio"; the owner's decision C, the +Architect's ruling (b), RULINGS-SD-DCAT-BATCH-2026-11-14-late.) + +Egeria has no "may I edit this?" API, so the answer is three-way and read as the user: +- **yes**: the user's own private draft (zoned exactly ``[]``); or a held artifact whose element the user + created (``versions.createdBy``) or owns (the Ownership classification names their user id). For a published + one, editing means a new draft from it (new_from) and later publishing a new version, which Egeria still + authorises on update. +- **no**: a built-in spec (pyegeria's own, not in Egeria; copy it to a new draft), or nothing the user can read in + Egeria (absent, or not visible to them: Egeria doesn't tell those apart). +- **unknown**: everything else, e.g. a steward with rights through a role (Egeria exposes no rights source), an + owner named by something other than a user id, or a read that didn't give an answer. The Portal hides the link + on unknown and logs `why`; Studio still checks on open and at publish. + +Nothing is cached: every call reads as `client`. +""" +from __future__ import annotations + +from typing import Any, Dict, Optional + +from pyegeria.authoring.models import ANSWER_REFUSED, ANSWER_UNKNOWN, KINDS, AuthoringRefused + +YES, NO, UNKNOWN = "yes", "no", "unknown" +CAN_EDIT_ANSWERS = (YES, NO, UNKNOWN) + + +def _answer(answer: str, why: str) -> Dict[str, str]: + return {"answer": answer, "why": why} + + +def _header(el: Dict[str, Any]) -> Dict[str, Any]: + return (el or {}).get("elementHeader") or {} + + +def _owner(el: Dict[str, Any]) -> Optional[Dict[str, Any]]: + """The Ownership classification's properties: its own header key in Egeria 6, else in the classification + lists (older shapes).""" + header = _header(el) + candidates = [header.get("ownership")] + for key in ("otherClassifications", "classifications"): + candidates += [c for c in (header.get(key) or []) if isinstance(c, dict) + and c.get("classificationName") == "Ownership"] + for c in candidates: + if isinstance(c, dict) and isinstance(c.get("classificationProperties"), dict): + return c["classificationProperties"] + return None + + +async def _read(kind: str, ref: str, client: Any) -> Optional[Dict[str, Any]]: + """The element as the user, None when Egeria holds none they can see. Raises AuthoringRefused (answer + unknown or refused) for a read that can't say, and lets any other error through.""" + from pyegeria.authoring.lifecycle import _by_qualified_name, _report_element + if kind == "spec": + from md_processing.v2.report import find_report_spec_element + return await find_report_spec_element(client, ref) + if kind == "report": + return await _report_element(client, ref) + return await _by_qualified_name(client, ref if ref.startswith("Dashboard::") else f"Dashboard::{ref}") + + +def _is_builtin_spec(label: str) -> bool: + from pyegeria.view.base_report_formats import lookup_report_spec + for prefix in ("ReportSpec::", "ReportType::"): + if label.startswith(prefix): + label = label[len(prefix):] + fs, _tier, _label = lookup_report_spec(f"builtin:{label}") + return fs is not None + + +async def async_can_edit(kind: str, ref: str, client: Any) -> Dict[str, str]: + """``{"answer": "yes"|"no"|"unknown", "why": }`` for `kind` ("spec", "report" or "sheet") named `ref` + (a label, display name or qualified name), read from Egeria as `client`'s user. Never raises for a read; + raises AuthoringRefused only for a bad kind, an empty ref, or no client.""" + if kind not in KINDS: + raise AuthoringRefused(f"can_edit works on one of {', '.join(KINDS)}, not {kind!r}.") + if not ref: + raise AuthoringRefused("can_edit needs the artifact's name.") + user = getattr(client, "user_id", None) if client is not None else None + if not user: + raise AuthoringRefused("can_edit reads Egeria as you: a signed-in client is needed.") + try: + el = await _read(kind, ref, client) + except AuthoringRefused as exc: + if getattr(exc, "answer", ANSWER_REFUSED) == ANSWER_UNKNOWN: + return _answer(UNKNOWN, f"Whether Egeria holds it couldn't be told: {exc}") + return _answer(UNKNOWN, str(exc)) # e.g. two Reports with that display name + except Exception as exc: # noqa: BLE001 -- a failed read is unknown, never "no" + return _answer(UNKNOWN, f"It couldn't be read ({type(exc).__name__}): nothing is assumed.") + + if el is None: + if kind == "spec" and _is_builtin_spec(ref): + return _answer(NO, f"{ref!r} is a built-in spec (pyegeria's own, not in Egeria): copy it to a new " + f"draft to change it.") + return _answer(NO, f"Egeria holds no {kind} {ref!r} that you can see.") + + from pyegeria.view.egeria_spec_cache import _zones + zones = _zones(el) + if zones == [user]: + return _answer(YES, "It's your private draft.") + versions = _header(el).get("versions") or {} + if versions.get("createdBy") == user: + return _answer(YES, "You created it." + (" Editing makes a new draft; publishing it makes a new version." + if user not in zones else "")) + owner = _owner(el) + if owner and owner.get("owner") == user and owner.get("ownerPropertyName") in (None, "", "userId"): + return _answer(YES, "You own it (Ownership). Editing makes a new draft; publishing it makes a new version.") + if owner and owner.get("owner") and owner.get("ownerPropertyName") not in (None, "", "userId"): + return _answer(UNKNOWN, f"Its owner is named by {owner.get('ownerPropertyName')} ({owner.get('owner')!r}), " + f"not a user id, so whether that's you isn't read here; Studio checks on open and " + f"at publish.") + return _answer(UNKNOWN, "You didn't create it and Ownership doesn't name you; Egeria exposes no other edit " + "rights (e.g. a steward's role), so Studio checks on open and at publish.") + + +def can_edit(kind: str, ref: str, client: Any) -> Dict[str, str]: + """Sync form of async_can_edit.""" + from pyegeria.core.async_runner import run_sync + return run_sync(async_can_edit(kind, ref, client)) diff --git a/pyegeria/authoring/preview.py b/pyegeria/authoring/preview.py index 2bcbf0e4..82c80471 100644 --- a/pyegeria/authoring/preview.py +++ b/pyegeria/authoring/preview.py @@ -45,10 +45,14 @@ def _cap(result: Dict[str, Any], max_rows: int, notes: List[str]) -> Dict[str, A rows = result.get("rows") elif isinstance(result, dict) and result.get("kind") == "json" and isinstance(result.get("data"), list): rows = result.get("data") - out = {"result": result, "capped": False, "shown": None, "notes": notes} + end = result.get("end") if isinstance(result, dict) else None + out = {"result": result, "capped": False, "shown": None, "notes": notes, "end": end} if isinstance(result, dict) and result.get("kind") == "empty": out["shown"] = 0 - notes.append("ran, and found nothing") + if end in (None, "confirmed"): # None: not a find (an analytic or chart result says its own absence) + notes.append("ran, and found nothing") + else: # pager step 3: "found nothing" only when the end is confirmed + notes.extend(result.get("notes") or ["nothing in what was read; can't rule out more"]) return out if rows is None: notes.append(f"ran; the result is {result.get('kind') if isinstance(result, dict) else type(result).__name__}, " @@ -64,6 +68,9 @@ def _cap(result: Dict[str, Any], max_rows: int, notes: List[str]) -> Dict[str, A elif len(rows) == max_rows: out["capped"] = True notes.append(f"capped: the run asked for {max_rows} rows and got {max_rows}; there may be more") + elif end not in (None, "confirmed"): # pager step 3: the end wasn't reached, whatever the count + out["capped"] = True + notes.append(f"{len(rows)} rows shown; Egeria didn't confirm there are no more") # end stays structured elif not rows: notes.append("ran, and found nothing") return out @@ -80,7 +87,9 @@ async def _run(label: str, fs, client: Any, output_format: str, params: Dict[str f"isn't registered") out = "DICT" params = {**params, "page_size": max_rows} - kwargs = dict(output_format=out, params=params, **_identity(client)) + # Pager step 3: page a list format fully, up to max_rows rows, so a filtered first page doesn't read as + # "found nothing" (a text format reads its first page and says so when that showed nothing). + kwargs = dict(output_format=out, params=params, page_to=max_rows, **_identity(client)) if out == "TABLE": kwargs["table_shape"] = "structured" result = await asyncio.to_thread(exec_report_spec, run_dict(label, fs, out), **kwargs) @@ -115,15 +124,16 @@ async def _sheet_for_resolver(draft: SheetDraft, client: Any, notes: List[str]): if p.text is not None: placements.append(EgeriaPlacement(ref=p.name, content=p.text, kind="text", **common)) continue + why = "no Report by this name that you can see" try: el = await _report_element(client, p.report) - except AuthoringRefused as exc: + except AuthoringRefused as exc: # two or more by that name, or an unknown read (never "no Report") el = None + why = str(exc) notes.append(f"placement {i} ({p.report}): {exc}") guid = (el or {}).get("elementHeader", {}).get("guid") placements.append(EgeriaPlacement(ref=p.report, kind="report" if guid else "missing", report_guid=guid, - note=None if guid else "no Report by this name that you can see", - **common)) + note=None if guid else why, **common)) return EgeriaDashboardSheet(name=draft.name, heading=draft.heading, description=draft.description, family=draft.family, placements=placements) diff --git a/pyegeria/authoring/publication.py b/pyegeria/authoring/publication.py index 49b69e32..447d967d 100644 --- a/pyegeria/authoring/publication.py +++ b/pyegeria/authoring/publication.py @@ -24,9 +24,22 @@ **Pins name publications**, not Egeria instance versions (those count every change, zones included). -**Exact target, never a twin.** Before any plan, the element is found by exact qualifiedName (a Report -without one, by exact display name), as the user, with a page of 25 (B5: never 1). More than one match, a -full page, or a failed read refuses: a wrongly created twin would split the history and orphan every pin. +**Exact target, never a twin.** Before any plan, the element is found by Egeria's unique-name read of its +qualifiedName, as the user: an exact server-side lookup, never a paged property search (pager step 5, the +exists-lookups ruling, 2026-11-14). Found -> update; Egeria's "not found" -> create; any other error or answer +refuses, as does an element of another type under that qualifiedName. A Report draft with no qualified name +is refused: one display-name match can't be proven the only one. A wrongly created twin would split the +history and orphan every pin. + +**Visibility.** The unique-name read is filtered by what the publisher may see (Egeria's +getBeanByUniqueName): an element zoned away from the publisher reads as "not found", so its publish plans a +create ("not held"). Egeria then refuses that create: qualifiedName is a unique property (Referenceable, +OpenMetadataTypesArchive1_2), and the generic create path (OpenMetadataAPIGenericHandler.createBeanInRepository +-> validateNewEntityRequest -> validateUniqueProperties -> validateUniqueProperty, read at Egeria 8de9572c4d) +looks the value up as the server's own user, so the caller's zones don't hide the existing element, and throws +"unique name in use". So the publish fails rather than making a twin; it doesn't update the element either. +Publish as someone who can see the element. (Read from the source for the generic-handler create path; not +every OMVS create has been traced to it.) **No silent overwrite.** apply re-resolves the target and refuses if its GUID or `publishedVersion` changed since the plan was made. @@ -37,10 +50,15 @@ """ from __future__ import annotations +import json from datetime import datetime, timezone from typing import Any, Dict, List, Optional, Tuple -from pyegeria.authoring.models import AuthoringRefused, ReportDraft, SpecDraft, as_draft +from loguru import logger + +from pyegeria.core._globals import NO_ELEMENTS_FOUND + +from pyegeria.authoring.models import ANSWER_UNKNOWN, AuthoringRefused, ReportDraft, SpecDraft, as_draft PUBLISHED_VERSION_KEY = "publishedVersion" PUBLISHED_AT_KEY = "publishedAt" @@ -148,61 +166,109 @@ def _is_not_found(exc: BaseException) -> bool: return isinstance(exc, PyegeriaNotFoundException) +def _type_names(el: Dict[str, Any]) -> List[str]: + t = (el.get("elementHeader") or {}).get("type") or {} + return [t.get("typeName"), *(t.get("superTypeNames") or [])] + + async def _find_exact(client: Any, type_name: str, prop: str, value: str) -> List[Dict[str, Any]]: + """The element publishing writes ([] when Egeria holds none), by Egeria's unique-name read: an exact, + server-side lookup by qualifiedName, with no client paging (the exists-lookups ruling, 2026-11-14). + + Publishing never pages a property search to choose create versus update: on a NULL_NOT_END path a + filtered page reads as "none", and a create would duplicate the element. So: + - found -> that element (update); Egeria's specific not-found -> [] (create); + - any other error -> unknown: refused with the words and the read, nothing planned; + - a target named by another property (a Report by displayName) -> refused: one match can't be proven the + only one, so the qualified name is needed.""" + if prop != "qualifiedName": + raise AuthoringRefused(f"Give the qualified name: Egeria can't confirm there's only one {type_name} with " + f"{prop} {value!r}, so which element publishing would write can't be told. " + f"Nothing was planned.") try: - found = await client.classification_manager._async_get_elements_by_property_value( - value, [prop], metadata_element_type=type_name, starts_with=False, ends_with=False, ignore_case=False, - page_size=LOOKUP_PAGE, graph_query_depth=0) + result = await client.classification_manager._async_get_element_by_unique_name(value) except Exception as exc: # noqa: BLE001 if _is_not_found(exc): return [] - raise AuthoringRefused(f"Couldn't find out whether Egeria already holds the {type_name} {value!r} " - f"({type(exc).__name__}). Publishing needs to know which element it writes, so " - f"nothing was planned; try again.") from exc - rows = [e for e in found if isinstance(e, dict) and e.get("elementHeader")] if isinstance(found, list) else [] - if len(rows) >= LOOKUP_PAGE: - raise AuthoringRefused(f"Egeria answered a full page ({LOOKUP_PAGE}) for the {type_name} {value!r}, so " - f"which element to write can't be told. Nothing was planned.") - return [e for e in rows if (e.get("properties") or {}).get(prop) == value] - - -#: Element history is read in pages of this size until an empty page (a short page isn't the last page: -#: Egeria's paging contract), up to HISTORY_MAX_PAGES pages. + logger.info(f"publication: unique-name read of {type_name} {value!r} failed: {type(exc).__name__}") + raise AuthoringRefused(f"Couldn't find out whether Egeria already holds the {type_name} {value!r} (the " + f"unique-name read of qualifiedName failed: {type(exc).__name__}). Publishing " + f"needs to know which element it writes, so nothing was planned; try again.", + answer=ANSWER_UNKNOWN) from exc + if result == NO_ELEMENTS_FOUND: # Egeria holds no element with that qualifiedName: absent + return [] + el = result if isinstance(result, dict) and result.get("elementHeader") else ( + result.get("element") if isinstance(result, dict) and isinstance(result.get("element"), dict) else None) + if el is None: # step 5 read-back R1: only "not found" is absent; any other answer is unknown, never a create + raise AuthoringRefused(f"Egeria's answer to the unique-name read of {type_name} {value!r} wasn't an element " + f"or 'not found' ({type(result).__name__}); nothing was planned.") + if type_name not in _type_names(el): + raise AuthoringRefused(f"Egeria holds a {_type_names(el)[0] or 'different element'} with qualifiedName " + f"{value!r}, not a {type_name}; publishing won't write over it. Nothing was planned.") + return [el] + + +#: Element history is read through the pager in pages of this size (pager step 4: >= 100, so the probes after a +#: null cover >= 300 positions), up to HISTORY_MAX_PAGES requests. HISTORY_PAGE = 100 HISTORY_MAX_PAGES = 50 HISTORY_CAPPED = ("its history is too long to read whole (more than {n} versions were returned and more may " "follow), so the entry that is each published version can't be told") -async def async_read_history(client: Any, guid: str) -> Tuple[List[Dict[str, Any]], Optional[str]]: +def _history_key(entry: Any) -> Any: + """A history entry's identity: every entry of one element carries the same GUID, so the GUID alone (the + pager's default key) would read the second version as a repeat. GUID + version + update time.""" + if isinstance(entry, dict): # the raw OpenMetadataElement (elementGUID, versions) or an elementHeader + header = entry.get("elementHeader") if isinstance(entry.get("elementHeader"), dict) else entry + guid = header.get("elementGUID") or header.get("guid") + versions = header.get("versions") or {} + if guid and (versions.get("version") is not None or versions.get("updateTime")): + return (guid, versions.get("version"), versions.get("updateTime")) + return "content:" + json.dumps(entry, sort_keys=True, default=str) + + +async def async_read_history(client: Any, guid: str, *, + trace: Optional[Dict[str, Any]] = None) -> Tuple[List[Dict[str, Any]], Optional[str]]: """(every history entry of `guid` in view shape, why it isn't whole or None), read as the user. - Paged (HISTORY_PAGE) until Egeria answers an empty page. The oldest entry stamped N is published version - N only if the history is whole: a page cap could hide the publication and leave a later edit as the - oldest one returned. So when the pages run out before an empty one (HISTORY_MAX_PAGES), or a read - fails, the reason is returned and the caller refuses or says "not checked"; it never resolves on part - of the history.""" + Paged through ``page_all`` (pager step 4, ruling (A)) at HISTORY_PAGE, up to HISTORY_MAX_PAGES requests. + The oldest entry stamped N is published version N only if the history is whole: a page cap could hide + the publication and leave a later edit as the oldest one returned. The read is acted on as whole when it + reached the end (a null, then empty probes on this suspect path); when a limit stopped it (capped, + stalled, the empty-run cap), or a read fails, the reason is returned and the caller refuses or says + "not checked"; it never resolves on part of the history. + + ``trace`` (if given) gets how the read ended (method, positions_read, end, end_reason), also logged, so a + later miss can be traced to the read.""" from pyegeria.core.element_shape import as_view_element - entries: List[Dict[str, Any]] = [] - start = 0 - for _ in range(HISTORY_MAX_PAGES): - try: - raw = await client.metadata_expert._async_get_element_history( - guid, oldest_first=False, start_from=start, page_size=HISTORY_PAGE) - except Exception as exc: # noqa: BLE001 -- said, not raised - if _is_not_found(exc): - return entries, None - return entries, f"its history couldn't be read ({type(exc).__name__})" - page = [e for e in raw if isinstance(e, dict)] if isinstance(raw, list) else [] - if not page: # "No elements found" or an empty list: the history is whole - return [as_view_element(e) for e in entries], None - entries += page - start += len(page) - return [as_view_element(e) for e in entries], HISTORY_CAPPED.format(n=len(entries)) - - -async def _history(client: Any, guid: str) -> Tuple[List[Dict[str, Any]], Optional[str]]: - return await async_read_history(client, guid) + from pyegeria.core.paging import async_page_all, log_end, paging_for + + async def fetch(start, size): + return await client.metadata_expert._async_get_element_history( + guid, oldest_first=False, start_from=start, page_size=size) + + try: + paged = await async_page_all(fetch, fetch_page_size=HISTORY_PAGE, max_pages=HISTORY_MAX_PAGES, + key=_history_key, **paging_for("get_element_history")) + except Exception as exc: # noqa: BLE001 -- said, not raised + if _is_not_found(exc): + return [], None + return [], f"its history couldn't be read ({type(exc).__name__})" + fields = log_end("publication.async_read_history", paged) + if trace is not None: + trace.update(fields) + entries = [as_view_element(e) for e in paged.items if isinstance(e, dict)] + if paged.reached_end: + return entries, None + if paged.end_reason == "max_pages": + return entries, HISTORY_CAPPED.format(n=len(entries)) + return entries, f"its history wasn't read whole ({paged.note})" + + +async def _history(client: Any, guid: str, trace: Optional[Dict[str, Any]] = None + ) -> Tuple[List[Dict[str, Any]], Optional[str]]: + return await async_read_history(client, guid, trace=trace) async def async_resolve_target(client: Any, draft: Any) -> Dict[str, Any]: @@ -231,14 +297,16 @@ async def _resolve(client: Any, target: Tuple[str, str, str, str]) -> Dict[str, el = matches[0] from pyegeria.view.egeria_spec_cache import _zones guid = (el.get("elementHeader") or {}).get("guid") - history, why = await _history(client, guid) if guid else ([], "it has no GUID") + history_read: Dict[str, Any] = {} + history, why = await _history(client, guid, history_read) if guid else ([], "it has no GUID") entries = ordered_entries(el, history) current = stamp_of(el) notes = [f"Edited-since check not done: {why}."] if why else [] edited = None if why else edited_since(entries, current) return {"resolved": True, "held": True, "guid": guid, "qualified_name": (el.get("properties") or {}).get("qualifiedName"), "zones": list(_zones(el)), "published_version": current, - "published_versions": published_versions(entries), "edited_since": edited, "notes": notes} + "published_versions": published_versions(entries), "edited_since": edited, "notes": notes, + "history_read": history_read} async def async_check_unchanged_target(client: Any, plan: Dict[str, Any]) -> None: diff --git a/pyegeria/authoring/used_by.py b/pyegeria/authoring/used_by.py new file mode 100644 index 00000000..632772a2 --- /dev/null +++ b/pyegeria/authoring/used_by.py @@ -0,0 +1,261 @@ +"""What uses this spec, Report or sheet? (P3, for Studio's impact view; DESIGN-P3-AUTHORING-USED-BY-2026-11-14, +APPROVED by the Architect's ruling (c), RULINGS-SD-DCAT-BATCH-2026-11-14-late.) + +``async_used_by(kind, label, client) -> {"items": [{"kind", "name", "qualified_name", "via"}], "complete", "notes"}`` + +Every read is live, as `client`'s user, through the pager; nothing is cached, and the Portal's shared consumer +index is never used (it's built as the sync identity, leaves zoned specs out, and covers folders only): + +- a spec's Reports: every Report the user can see (by type), matched on ``additionalProperties.reportSpec`` + (the label, or ``ReportSpec::``/``ReportType::`` + label); via ``"reportSpec"``; +- a spec's consumer folders: its element's CollectionMembership (the spec at end 2), kept to + ``ReportSpecFolder::Consumer::`` folders; via ``"CollectionMembership"``; +- a Report's sheets: DataSetContent from a Dashboard to the Report (the Report at end 2); via ``"DataSetContent"``; +- a sheet's consumer folders: ``Dashboard::``'s CollectionMembership, kept to ``DashboardFolder::Consumer::``. + +``via`` is a closed set of machine keys (``VIA_KEYS``): ``"reportSpec"`` (runs this spec), +``"CollectionMembership"`` (in a consumer folder), ``"DataSetContent"`` (placed on a sheet). They're Egeria's own +names; callers (Studio) map them to words, and pyegeria carries no UI words (the Architect's P3 open point 2). + +Resolving the artifact comes first, through Egeria's exact unique-name read (P3-1): "absent" only on Egeria's +specific not-found; anything else is "unknown". An unresolvable artifact raises AuthoringRefused with ``.answer`` +``"absent"`` or ``"unknown"``; nothing is scanned. One exception: a built-in spec that Egeria doesn't hold still +gets its Reports listed, with a note. + +Call it on demand (when the user opens "Used by"), never per keystroke or on page load: a spec costs one bounded +Report scan (at most 2,000 Reports) plus a related read. ``complete`` is True only when every read ended +confirmed. A read that fails, or isn't confirmed, is said in ``notes`` and makes ``complete`` False: an empty +``items`` is then "nothing found in what Egeria returned", never "used by none". What the user can't see is not +listed: the answer is their view. +""" +from __future__ import annotations + +from typing import Any, Dict, List, Optional + +from pyegeria.authoring.models import ANSWER_ABSENT, ANSWER_UNKNOWN, AuthoringRefused + +REPORT_SCAN_PAGE = 100 +REPORT_SCAN_MAX_PAGES = 20 # at most 2000 Reports per call (the Architect's cost guard) +RELATED_PAGE = 100 +RELATED_MAX_PAGES = 10 +USED_BY_KINDS = ("spec", "report", "sheet") +VIA_KEYS = ("reportSpec", "CollectionMembership", "DataSetContent") # closed: Studio's word map tests against it +SPEC_PREFIX = "ReportSpec::" +LEGACY_SPEC_PREFIX = "ReportType::" # the legacy loader's form only: remove with it in pyegeria 7 + + +def spec_label_of(value: Any) -> str: + """A reportSpec value (or a spec name) as a bare label: one ``ReportSpec::`` or legacy ``ReportType::`` prefix + stripped, nothing else changed. Matching is exact and case-sensitive on the result (P3-2).""" + value = value if isinstance(value, str) else "" + for prefix in (SPEC_PREFIX, LEGACY_SPEC_PREFIX): + if value.startswith(prefix): + return value[len(prefix):] + return value +AS_YOU = "Read as you: what you can't see in Egeria isn't listed." + + +def _props(el: Dict[str, Any]) -> Dict[str, Any]: + return (el or {}).get("properties") or {} + + +def _guid(el: Dict[str, Any]) -> Optional[str]: + return ((el or {}).get("elementHeader") or {}).get("guid") + + +def _type(el: Dict[str, Any]) -> str: + return (((el or {}).get("elementHeader") or {}).get("type") or {}).get("typeName", "") + + +def _related(entry: Dict[str, Any]) -> Dict[str, Any]: + return entry.get("relatedElement") if isinstance(entry.get("relatedElement"), dict) else entry + + +class _Answer: + def __init__(self): + self.items: List[Dict[str, Any]] = [] + self.notes: List[str] = [] + self.complete = True + self._seen = set() + + def add(self, kind: str, name: str, qn: Optional[str], via: str) -> None: + key = (kind, qn or name, via) + if key not in self._seen: + self._seen.add(key) + self.items.append({"kind": kind, "name": name, "qualified_name": qn, "via": via}) + + def unconfirmed(self, words: str) -> None: + self.complete = False + self.notes.append(words) + + def result(self) -> Dict[str, Any]: + notes = list(self.notes) + if not self.items and not self.complete: + notes.append("Nothing found in what Egeria returned; that isn't \"used by none\".") + return {"items": self.items, "complete": self.complete, "notes": notes + [AS_YOU]} + + +async def _paged(fetch, method: str, client: Any, page: int, max_pages: int): + from pyegeria.core.paging import async_page_all, async_paging_for, log_end + paged = await async_page_all(fetch, fetch_page_size=page, max_pages=max_pages, + **await async_paging_for(method, client)) + log_end(f"used_by.{method}", paged) + return paged + + +async def _reports_running(spec_label: str, client: Any, out: _Answer) -> None: + from pyegeria.core.paging import at_least_note + + async def fetch(start, size): + return await client.classification_manager._async_get_elements( + "Report", start_from=start, page_size=size, graph_query_depth=0) + try: + paged = await _paged(fetch, "get_elements", client, REPORT_SCAN_PAGE, REPORT_SCAN_MAX_PAGES) + except Exception as exc: # noqa: BLE001 -- said, never "none" + out.unconfirmed(f"The Reports couldn't be read ({type(exc).__name__}), so which run this spec isn't known.") + return + hits = [e for e in paged.items if isinstance(e, dict) and _type(e) == "Report" + and spec_label_of((_props(e).get("additionalProperties") or {}).get("reportSpec")) == spec_label] + for e in hits: + p = _props(e) + out.add("report", p.get("displayName") or p.get("qualifiedName"), p.get("qualifiedName"), "reportSpec") + if not paged.complete: + out.unconfirmed(at_least_note(paged, "Reports that run this spec", len(hits))) + + +async def _members_of(guid: str, client: Any, prefix: str, what: str, out: _Answer) -> None: + """The consumer folders `guid` is a member of (CollectionMembership, the member at end 2).""" + from pyegeria.core._exceptions import PyegeriaNotFoundException + from pyegeria.core.paging import at_least_note + + async def fetch(start, size): + try: + return await client.classification_manager._async_get_related_elements( + guid, "CollectionMembership", start_at_end=2, start_from=start, page_size=size, + graph_query_depth=0) + except PyegeriaNotFoundException: + return None + try: + paged = await _paged(fetch, "get_related_elements", client, RELATED_PAGE, RELATED_MAX_PAGES) + except Exception as exc: # noqa: BLE001 + out.unconfirmed(f"The folders holding this {what} couldn't be read ({type(exc).__name__}).") + return + n = 0 + for entry in paged.items: + folder = _related(entry) if isinstance(entry, dict) else {} + qn = str(_props(folder).get("qualifiedName") or "") + if qn.startswith(prefix): + n += 1 + out.add("folder", qn[len(prefix):], qn, "CollectionMembership") + if not paged.complete: + out.unconfirmed(at_least_note(paged, f"consumer folders holding this {what}", n)) + + +async def _sheets_placing(report_guid: str, client: Any, out: _Answer) -> None: + from pyegeria.core._exceptions import PyegeriaNotFoundException + from pyegeria.core.paging import at_least_note + from pyegeria.view.dashboards import DASHBOARD_QN_PREFIX, sheet_name_from_dashboard_qn + + async def fetch(start, size): + try: + return await client.classification_manager._async_get_related_elements( + report_guid, "DataSetContent", start_at_end=2, start_from=start, page_size=size, + graph_query_depth=0) + except PyegeriaNotFoundException: + return None + try: + paged = await _paged(fetch, "get_related_elements", client, RELATED_PAGE, RELATED_MAX_PAGES) + except Exception as exc: # noqa: BLE001 + out.unconfirmed(f"The sheets placing this Report couldn't be read ({type(exc).__name__}).") + return + n = 0 + for entry in paged.items: + dash = _related(entry) if isinstance(entry, dict) else {} + qn = str(_props(dash).get("qualifiedName") or "") + if qn.startswith(DASHBOARD_QN_PREFIX): + n += 1 + out.add("sheet", sheet_name_from_dashboard_qn(qn), qn, "DataSetContent") + if not paged.complete: + out.unconfirmed(at_least_note(paged, "sheets placing this Report", n)) + + +async def _unique(client: Any, qn: str, what: str): + """Egeria's exact unique-name read of `qn`: the element, or None on Egeria's specific not-found. Unknown raises + AuthoringRefused(answer="unknown"), never None.""" + from pyegeria.core.unique_name import async_by_unique_name + found = await async_by_unique_name(client, qn) + if found.answer == "unknown": + raise AuthoringRefused(f"Couldn't confirm whether Egeria holds {what}: {found.reason or 'no clear answer'}", + answer=ANSWER_UNKNOWN) + return found.element if found.found else None + + +def _is_builtin_spec(label: str) -> bool: + from pyegeria.view.base_report_formats import lookup_report_spec + try: + fs, _tier, _label = lookup_report_spec(f"builtin:{label}") + except Exception: # noqa: BLE001 -- not known as built in + return False + return fs is not None + + +async def async_used_by(kind: str, label: str, client: Any) -> Dict[str, Any]: + """What uses `kind` `label`, as `client`'s user. Never raises for a failed read of what USES it (that's in + ``notes``, and ``complete`` is False). Raises AuthoringRefused for a bad kind or no client, and, with ``answer`` + ``"absent"`` or ``"unknown"``, for an artifact that can't be resolved (P3-1).""" + from pyegeria.authoring.lifecycle import _report_element + from pyegeria.authoring.serialize import DASHBOARD_FOLDER_PREFIX, SPEC_FOLDER_PREFIX + if kind not in USED_BY_KINDS: + raise AuthoringRefused(f"used_by works on one of {', '.join(USED_BY_KINDS)}, not {kind!r}.") + if not label: + raise AuthoringRefused("used_by needs the artifact's name.") + if client is None or not getattr(client, "user_id", None): + raise AuthoringRefused("used_by reads Egeria as you: a signed-in client is needed.") + out = _Answer() + + if kind == "spec": + bare = spec_label_of(label) + el = None + for qn in (f"{SPEC_PREFIX}{bare}", f"{LEGACY_SPEC_PREFIX}{bare}"): + el = await _unique(client, qn, f"a report spec {qn}") + if el: + break + if el is None and not _is_builtin_spec(bare): + raise AuthoringRefused(f"Egeria holds no report spec named {bare!r} that you can see.", + answer=ANSWER_ABSENT) + await _reports_running(bare, client, out) + if el is None: + out.notes.append(f"Egeria holds no report spec {bare!r} that you can see (it's built in), so no " + f"consumer folder holds it for you.") + elif _guid(el): + await _members_of(_guid(el), client, SPEC_FOLDER_PREFIX, "spec", out) + return out.result() + + if kind == "report": + el = await _unique(client, label, f"a Report {label}") + if not (el and _type(el) == "Report"): + try: + el = await _report_element(client, label) # by display name; unknown raises (answer unknown) + except AuthoringRefused: + raise + except Exception as exc: # noqa: BLE001 -- couldn't tell: unknown, never absent + raise AuthoringRefused(f"Couldn't confirm whether Egeria holds a Report named {label!r} " + f"({type(exc).__name__}).", answer=ANSWER_UNKNOWN) from exc + if not el: + raise AuthoringRefused(f"Egeria holds no Report named {label!r} that you can see.", answer=ANSWER_ABSENT) + await _sheets_placing(_guid(el), client, out) + return out.result() + + qn = label if label.startswith("Dashboard::") else f"Dashboard::{label}" + el = await _unique(client, qn, f"a dashboard sheet {qn}") + if not el: + raise AuthoringRefused(f"Egeria holds no dashboard sheet {label!r} ({qn}) that you can see.", + answer=ANSWER_ABSENT) + await _members_of(_guid(el), client, DASHBOARD_FOLDER_PREFIX, "sheet", out) + return out.result() + + +def used_by(kind: str, label: str, client: Any) -> Dict[str, Any]: + """Sync form of async_used_by.""" + from pyegeria.core.async_runner import run_sync + return run_sync(async_used_by(kind, label, client)) diff --git a/pyegeria/authoring/versions.py b/pyegeria/authoring/versions.py index 9029df54..1b55948e 100644 --- a/pyegeria/authoring/versions.py +++ b/pyegeria/authoring/versions.py @@ -16,7 +16,7 @@ from typing import Any, Dict, List, Optional, Tuple -from pyegeria.authoring.models import AuthoringRefused +from pyegeria.authoring.models import ANSWER_ABSENT, ANSWER_UNKNOWN, AuthoringRefused def as_element(item: Dict[str, Any]) -> Dict[str, Any]: @@ -44,12 +44,13 @@ async def async_spec_versions(client: Any, label: str) -> Tuple[Dict[str, Any], from md_processing.v2.report import find_report_spec_element current = await find_report_spec_element(client, label) if not current: - raise AuthoringRefused(f"Egeria holds no report spec {label!r} that you can see.") + raise AuthoringRefused(f"Egeria holds no report spec {label!r} that you can see.", answer=ANSWER_ABSENT) guid = (current.get("elementHeader") or {}).get("guid") from pyegeria.authoring.publication import async_read_history versions, why = await async_read_history(client, guid) if why: - raise AuthoringRefused(f"The published versions of spec {label!r} can't be told: {why}.") + raise AuthoringRefused(f"The published versions of spec {label!r} can't be told: {why}.", + answer=ANSWER_UNKNOWN) return current, versions diff --git a/pyegeria/core/_base_platform_client.py b/pyegeria/core/_base_platform_client.py index e8f65762..811616d0 100644 --- a/pyegeria/core/_base_platform_client.py +++ b/pyegeria/core/_base_platform_client.py @@ -7,6 +7,7 @@ """ +from pyegeria.core._no_password import NO_PASSWORD, carried_password, password_as_given, resolve_password import asyncio import inspect import json @@ -166,7 +167,7 @@ def __init__( self.user_id = user_id or settings.User_Profile.user_name if self.user_id: _validate_url_path_safe(self.user_id, "user_id") - self.user_pwd = user_pwd or settings.User_Profile.user_pwd + self.user_pwd, self.use_default_password = resolve_password(user_pwd) # NO_PASSWORD: opt out self.page_size = page_size or max_paging_size self.timeout = timeout or settings.Debug.timeout_seconds or 30 self.token_src = token_src diff --git a/pyegeria/core/_base_server_client.py b/pyegeria/core/_base_server_client.py index 20d0132d..5eb064f7 100644 --- a/pyegeria/core/_base_server_client.py +++ b/pyegeria/core/_base_server_client.py @@ -7,6 +7,7 @@ """ +from pyegeria.core._no_password import NO_PASSWORD, carried_password, password_as_given, resolve_password import asyncio import inspect import json @@ -40,6 +41,13 @@ +def _is_server_not_available(json_response: dict) -> bool: + """Egeria's "the OMAG Server / service is not available" 404 (OMAG-MULTI-TENANT-404-001/-002): a request to a + server or service that isn't running there, not a missing element.""" + message_id = str((json_response or {}).get("exceptionErrorMessageId") or "") + return message_id.startswith("OMAG-MULTI-TENANT-404") + + def _is_authorization_failure(json_response: dict) -> bool: """Whether an Egeria-wrapped error is really an authorization failure. @@ -110,7 +118,12 @@ def __init__( self.user_id = user_id or settings.User_Profile.user_name if self.user_id: _validate_url_path_safe(self.user_id, "user_id") - self.user_pwd = user_pwd or settings.User_Profile.user_pwd + self.user_pwd, self.use_default_password = resolve_password(user_pwd) # NO_PASSWORD: opt out + try: # whose report-spec cache a read outside a request is for (see _client_identities) + from pyegeria.core._client_identities import note_credentials + note_credentials(self.user_id, self.platform_url, self.server_name) + except Exception: # noqa: BLE001 -- recording must never break a client + pass self.page_size = page_size or max_paging_size self.token_src = token_src self.token = token @@ -679,6 +692,11 @@ async def _async_make_request( related_http_code = json_response.get("relatedHTTPCode", 0) if related_http_code == 200: return response + elif related_http_code == 404 and _is_server_not_available(json_response): + # OMAG-MULTI-TENANT-404-*: the server (or a service on it) isn't available, e.g. a + # wrong view server name. That is not "the element isn't there": raised as an API + # error so no caller's not-found handling reads it as absent (pager step 6). + raise PyegeriaAPIException(response, context, additional_info=json_response) elif related_http_code == 404: raise PyegeriaNotFoundException(response, context, additional_info=json_response) elif related_http_code in (401, 403) and _is_authorization_failure(json_response): diff --git a/pyegeria/core/_client_identities.py b/pyegeria/core/_client_identities.py new file mode 100644 index 00000000..709c978a --- /dev/null +++ b/pyegeria/core/_client_identities.py @@ -0,0 +1,69 @@ +"""The Egeria credentials this process has made view-server clients with, and whether it serves many users. + +The report-spec cache reads its EGERIA tier for a credential identity. Inside `exec_report_spec`, the async +runner and the dashboard resolver, that identity is set from the request's client. A read made outside them +(listing, describing, `report_spec_source`) has no client, so it uses the identity of this process's clients: +when the process has seen exactly one, that one. A process that has seen more than one is serving several +users, and no private cache loads outside a request's own identity there. + +A process that serves many users declares it (`declare_multi_user()`, or PYEGERIA_MULTI_USER=1): the Portal, EA, +Studio and the MCP HTTP server. Counting identities is only a backstop there: a server builds clients as its +service user and then swaps `user_id` to the signed-in user's (the Portal's use_token), so the first identity +recorded is the service user's until a second one is seen. A declared process never takes a fallback identity. + +This module only records; `pyegeria.view.egeria_spec_cache.current_spec_identity` decides. +""" +from __future__ import annotations + +import os +import threading +from typing import Any, List, Optional, Tuple + +_LOCK = threading.Lock() +_SEEN: List[Tuple[str, Optional[str], Optional[str]]] = [] +_MULTI_USER = False +MULTI_USER_ENV = "PYEGERIA_MULTI_USER" +_MAX_SEEN = 16 # the reader only needs "one" or "more than one"; a long-running multi-user process stops growing + + +def note_credentials(user_id: Any, platform_url: Any, view_server: Any) -> None: + """Record a (user, platform url, view server) this process makes Egeria calls with. Never raises.""" + if not isinstance(user_id, str) or not user_id.strip() or user_id.strip() == "?": + return + entry = (user_id.strip(), platform_url if isinstance(platform_url, str) and platform_url else None, + view_server if isinstance(view_server, str) and view_server else None) + with _LOCK: + if entry not in _SEEN and len(_SEEN) < _MAX_SEEN: + _SEEN.append(entry) + + +def note_client(client: Any) -> None: + """`note_credentials` for a client object (its view_server, else its server_name).""" + note_credentials(getattr(client, "user_id", None), getattr(client, "platform_url", None), + getattr(client, "view_server", None) or getattr(client, "server_name", None)) + + +def seen_credentials() -> List[Tuple[str, Optional[str], Optional[str]]]: + with _LOCK: + return list(_SEEN) + + +def reset_credentials() -> None: + """Forget every noted credential and any multi-user declaration (tests).""" + global _MULTI_USER + with _LOCK: + _SEEN.clear() + _MULTI_USER = False + + +def declare_multi_user(multi_user: bool = True) -> None: + """Declare that this process serves more than one Egeria user (call at startup). Then no read outside a + request's own identity (`spec_identity`) takes a fallback identity: no private cache loads for it and no + fetched copy is served to it.""" + global _MULTI_USER + _MULTI_USER = bool(multi_user) + + +def is_multi_user() -> bool: + """Declared multi-user: `declare_multi_user()` or PYEGERIA_MULTI_USER=1/true/yes.""" + return _MULTI_USER or os.getenv(MULTI_USER_ENV, "").strip().lower() in ("1", "true", "yes") diff --git a/pyegeria/core/_helper_client.py b/pyegeria/core/_helper_client.py new file mode 100644 index 00000000..d308d09a --- /dev/null +++ b/pyegeria/core/_helper_client.py @@ -0,0 +1,85 @@ +"""`helper_client`: the one way pyegeria builds a client that acts for another client (Architect C3). + +pyegeria builds clients for a caller in several places: EgeriaTech's (and Egeria's, EgeriaConfig's) lazily built +sub-clients, and helpers such as ClassificationExplorer's CollectionManager, AutomatedCuration's +GovernanceEngine, DataDiscovery's AssetMaker, the spec cache's CollectionManager, the unique-name reader's +ClassificationExplorer, and the executor's fallback client. Each copied the caller's `user_pwd` and `token` +attributes. Two things went wrong: + +- `set_bearer_token` writes only the Authorization headers, never `self.token`, so a caller given its token + that way passed `token=None`: the helper had no token and signed in with the password it held; +- a raw `user_pwd` of None is filled by the helper's constructor with the configured default (6.x), so a + token-only caller's helper held the demo password (the 10-28 shape). + +`helper_client(cls, parent, **kw)` builds `cls` on the parent's platform and view server, as the parent's +user, with: + +- the parent's BARE token, read from its Authorization header (or its `token`), set once through + `set_bearer_token` (so exactly one "Bearer " prefix); +- the password: `carried_password(parent)` (the parent's own password, or NO_PASSWORD when it opted out or + holds none; never the configured default). Under a token the parent didn't mint for itself, NO_PASSWORD: such + a token is never renewed (see BaseServerClient._owns_token), so a password could only be used to sign in again; +- `_token_minted_for` copied only when the parent minted its token for its own user, so the helper renews the + token exactly when the parent would. +""" +from __future__ import annotations + +from typing import Any, Optional + +from pyegeria.core._no_password import NO_PASSWORD, carried_password + + +def _text(value: Any) -> Optional[str]: + return (value.strip() or None) if isinstance(value, str) else None + + +def bare_token(parent: Any) -> Optional[str]: + """The parent's bearer token without any "Bearer " prefix, or None when it holds none. Read from its + Authorization header (`get_token`) first, which `set_bearer_token` keeps current, then its `token`.""" + token = None + # A facade (it holds a _subclient_map) is asked only through its class's own get_token: one that delegates + # unknown names to its sub-clients (EgeriaConfig's __getattr__) would build a sub-client to answer, and + # that sub-client is built by this function. + if "_subclient_map" in getattr(parent, "__dict__", {}): + method = getattr(type(parent), "get_token", None) + getter = (lambda: method(parent)) if callable(method) else None + else: + getter = getattr(parent, "get_token", None) + if callable(getter): + try: + token = _text(getter()) + except Exception: # noqa: BLE001 -- no Authorization header + token = None + if not token: + token = _text(getattr(parent, "token", None)) + while token and token.lower().startswith("bearer "): + token = _text(token[7:]) + return token + + +def parent_owns_token(parent: Any) -> bool: + """The parent minted its token for its own user (it would renew it on a 401).""" + minted = getattr(parent, "_token_minted_for", None) + return bool(minted) and minted == getattr(parent, "user_id", None) + + +def helper_password(parent: Any, token: Optional[str]) -> Any: + """The `user_pwd` a helper acting for `parent` is built with (see the module docstring).""" + if token and not parent_owns_token(parent): + return NO_PASSWORD + return carried_password(parent) + + +def helper_client(cls: Any, parent: Any, *, view_server: Optional[str] = None, **kwargs: Any) -> Any: + """`cls` built to act for `parent`: its platform, view server and user; its bare token set once; its own + password or none, never the configured default; `_token_minted_for` copied only for a self-minted token. + `kwargs` go to `cls` (timeout=..., legacy_fallback=...).""" + server = view_server or getattr(parent, "view_server", None) or getattr(parent, "server_name", None) + token = bare_token(parent) + client = cls(server, platform_url=getattr(parent, "platform_url", None), user_id=getattr(parent, "user_id", None), + user_pwd=helper_password(parent, token), **kwargs) + if token: + client.set_bearer_token(token) + if parent_owns_token(parent): + client._token_minted_for = parent._token_minted_for + return client diff --git a/pyegeria/core/_no_password.py b/pyegeria/core/_no_password.py new file mode 100644 index 00000000..a44e3663 --- /dev/null +++ b/pyegeria/core/_no_password.py @@ -0,0 +1,63 @@ +"""`NO_PASSWORD`: an explicit opt-out of the configured default password (owner ruling 2026-11-14, "Opt-out in +6.x"). + +In pyegeria 6.x a client given `user_pwd=None` (or no password) takes the configured password, +settings.User_Profile.user_pwd; CLIs pass `args.password`, None when the flag is omitted, and rely on that, so +None keeps that meaning. A caller that must never sign in with the configured password (a token-only client +acting for a signed-in user: the Portal, Studio, EA) passes `user_pwd=NO_PASSWORD` instead: + +- the client holds no password (`user_pwd` is None) and records `use_default_password = False`; +- `EgeriaTech` (and `EgeriaClient`, `EgeriaConfig`) carry the opt-out to every sub-client they build, and + pyegeria's internal helper clients carry it from the client they act for (`carried_password`); +- so the settings default password is never used: an expired token fails with a 401, and the one 401 auto-retry + can't sign in again as the configured (demo) user. + +pyegeria 7 drops the default password altogether. +""" +from __future__ import annotations + +from typing import Any, Optional, Tuple + + +class _NoPassword: + __slots__ = () + + def __repr__(self) -> str: + return "NO_PASSWORD" + + def __bool__(self) -> bool: + return False + + def __reduce__(self): + return "NO_PASSWORD" + + +NO_PASSWORD: Any = _NoPassword() + + +def resolve_password(user_pwd: Any) -> Tuple[Optional[str], bool]: + """(the password a client holds, whether it may use the configured default). NO_PASSWORD gives + (None, False); anything else keeps its 6.x meaning: a falsy value takes the configured password.""" + if user_pwd is NO_PASSWORD: + return None, False + if user_pwd: + return user_pwd, True + from pyegeria.core.config import settings + return settings.User_Profile.user_pwd, True + + +def password_as_given(user_pwd: Any) -> Optional[str]: + """`user_pwd` as the caller gave it, with NO_PASSWORD read as None (for classes that store it as given).""" + return None if user_pwd is NO_PASSWORD else user_pwd + + +def carried_password(client: Any) -> Any: + """The `user_pwd` to build a sub-client or helper client with, acting for `client`: the password `client` + holds, or NO_PASSWORD when it holds none (it opted out, or its password was cleared after construction, as + the Portal's use_token and Studio's/EA's scrub do). A child never holds a password its parent doesn't, so + it never falls back to the configured default (Architect C1). A CLI's client holds the configured password + already (None filled it at construction), so CLIs are unaffected.""" + password = getattr(client, "user_pwd", None) + if getattr(client, "use_default_password", True) is False or not password: + return NO_PASSWORD + return password diff --git a/pyegeria/core/_server_client.py b/pyegeria/core/_server_client.py index a8db273a..c56df169 100644 --- a/pyegeria/core/_server_client.py +++ b/pyegeria/core/_server_client.py @@ -25,7 +25,8 @@ from pyegeria.core.request_filters import deprecated_names from pyegeria.core._base_server_client import BaseServerClient from pyegeria.core._exceptions import ( - PyegeriaConnectionException, PyegeriaInvalidParameterException, PyegeriaException, PyegeriaErrorCode + PyegeriaConnectionException, PyegeriaInvalidParameterException, PyegeriaException, PyegeriaErrorCode, + PyegeriaNotFoundException, ) from pyegeria.core._globals import max_paging_size, NO_ELEMENTS_FOUND, default_timeout, COMMENT_TYPES from pyegeria.view.base_report_formats import get_report_spec_match @@ -272,6 +273,7 @@ async def __async_get_guid__( tech_type: Optional[str] = None, organization_name: Optional[str] = None, for_lineage: bool = False, + strict_errors: bool = False, ) -> str: """Helper function to return a server_guid - one of server_guid, qualified_name or display_name should contain information. If all are None, an exception will be thrown. If all contain @@ -316,8 +318,13 @@ async def __async_get_guid__( guid_found = result.json().get("guid", NO_ELEMENTS_FOUND) if guid_found != NO_ELEMENTS_FOUND: return guid_found + except PyegeriaNotFoundException: + pass # not held under that qualifiedName: try the display name except Exception: - pass + # strict_errors (pager step 5): a transport/auth/timeout error is not "not found", so it is raised, + # never folded into a fallback that may answer "not found" (and a Create-or-Update create). + if strict_errors: + raise if display_name: if property_name == "resourceName": @@ -542,7 +549,7 @@ async def _async_get_relationships_with_property_value( response: Response = await self._async_make_request( "POST", url, body_slimmer(body), timeout=timeout ) - rels = response.json().get("relationships", NO_ELEMENTS_FOUND) + rels = self._elements_noting_page_cap(response, NO_ELEMENTS_FOUND, key="relationships") if type(rels) is list: if len(rels) == 0: return NO_ELEMENTS_FOUND @@ -844,6 +851,32 @@ def get_guid_for_name( ) return response + async def _async_lookup_guid_for_name( + self, name: str, property_name: list[str] = ["qualifiedName", "displayName", "resourceName", "identifier"], + type_name: str = None, max_pages: int = 10): + """The three-way form of `_async_get_guid_for_name` (pager step 5, the exists-lookups ruling): a + `pyegeria.core.paging.Exists` -- "found" (an element whose `property_name` equals `name` exactly was + read; `matches` holds them), "absent" (the read reached the end) or "unknown". The by-property-value + read is NULL_NOT_END, so it is never "absent" before Egeria returns [] for a filtered page: a write + guard must treat "unknown" as "don't create; say so". It stops at the first page with a match, so it + never proves the name unique; more than one match read raises, as `_async_get_guid_for_name` does. + `_async_get_guid_for_name`'s own return is unchanged.""" + from pyegeria.core.paging import async_exists + + async def fetch(start, size): + return await self._async_get_elements_by_property_value(name, property_name, type_name, + start_from=start, page_size=size) + + def match(el): + props = (el.get("properties") or {}) if isinstance(el, dict) else {} + return any(props.get(p) == name for p in property_name) + + found = await async_exists(fetch, match, method="_async_get_elements_by_property_value", + max_pages=max_pages, client=self) # trusted on a #9453 platform (step 6) + if len(found.matches) > 1: + raise PyegeriaException(context={"issue": "Multiple elements found for supplied name!"}) + return found + async def _async_get_element_by_guid_(self, element_guid: str) -> dict | str: """ Simplified, internal version of get_element_by_guid found in Classification Manager. @@ -1921,7 +1954,19 @@ def _generate_formatted_output( """ if output_format.upper() == "JSON": return elements - + # The EGERIA tier is read for this client's credential, not the process fallback (spec-cache T2). + from pyegeria.view.egeria_spec_cache import as_client + with as_client(self): + return ServerClient._generate_formatted_output_as_client( + self, elements, query_string, entity_type, output_format, extract_properties_func, report_spec, **kwargs) + + def _spec_identity(self): + """A context in which EGERIA-tier reads are for this client's credential (spec-cache T2).""" + from pyegeria.view.egeria_spec_cache import as_client + return as_client(self) + + def _generate_formatted_output_as_client(self, elements, query_string, entity_type, output_format, + extract_properties_func, report_spec, **kwargs): # Resolve output formats structure output_formats = resolve_output_formats( entity_type, @@ -3138,9 +3183,14 @@ async def _async_add_journal_entry(self, note_log_qn: Optional[str] = None, elem """ note_log_guid = None element_guid = None - # If a note_log_qn has been provided, look up its GUID + # If a note_log_qn has been provided, look up its GUID. Pager step 5: a write guard (not found -> a new + # NoteLog is created), so it uses the three-way lookup: "unknown" never creates a second NoteLog. if note_log_qn: - note_log_guid = await self._async_get_guid_for_name(note_log_qn, ["qualifiedName"],'NoteLog' ) if note_log_qn else None + found = await self._async_lookup_guid_for_name(note_log_qn, ["qualifiedName"], 'NoteLog') + if found.answer == "unknown": + context = {"reason": f"{found.words(f'NoteLog {note_log_qn!r}')}; no new NoteLog was created"} + raise PyegeriaException(error_code=PyegeriaErrorCode.VALIDATION_ERROR, context=context) + note_log_guid = found.matches[0]["elementHeader"]["guid"] if found.found else None # If we need to create a new note_log, then use the qualified name from the element_qn parameter, if provided. if note_log_guid is None or note_log_guid == NO_ELEMENTS_FOUND: @@ -3149,10 +3199,13 @@ async def _async_add_journal_entry(self, note_log_qn: Optional[str] = None, elem note_log_display_name = f"NoteLog-{datetime.now().strftime('%Y-%m-%d %H:%M')}" else: - element_guid = await self._async_get_guid_for_name(element_qn, ["qualifiedName"]) - if element_guid is None or element_guid == NO_ELEMENTS_FOUND: - context = { "reason" : "The specified associated element was not found"} + found = await self._async_lookup_guid_for_name(element_qn, ["qualifiedName"]) + if not found.found: + reason = ("The specified associated element was not found" if found.answer == "absent" + else found.words(f"the associated element {element_qn!r}")) + context = {"reason": reason} raise PyegeriaException(error_code = PyegeriaErrorCode.VALIDATION_ERROR, context = context) + element_guid = found.matches[0]["elementHeader"]["guid"] note_log = await self._async_create_note_log(element_guid = element_guid, display_name = note_log_display_name) note_log_guid = note_log @@ -6519,17 +6572,25 @@ def validate_update_relationship_request(self, body: dict | UpdateRelationshipRe validated_body = None return validated_body - def _elements_noting_page_cap(self, response, default): - """The `elements` of a list answer. When a caller is recording (`recording_list_requests`), the - answer's facts go into the caller's own record, read back from the request that was sent, so - every list helper is covered: `cap` = (shown, page_size, start_from) when the answer filled the - page size, plus the graph depth and relationship filters sent. Nothing is kept on the client: a - client shared by concurrent runs (dashboard tiles) never mixes one run's cap with another's - (fix-now F8, read-back C1).""" - elements = response.json().get("elements", default) + def _elements_noting_page_cap(self, response, default, key: str = "elements"): + """The `elements` (or `key`, e.g. "relationships") of a list answer. When a caller is recording + (`recording_list_requests`), the answer's facts go into the caller's own record, read back from the + request that was sent, so every list helper is covered: `cap` = (shown, page_size, start_from) when + the answer filled the page size, plus the graph depth and relationship filters sent. Nothing is kept + on the client: a client shared by concurrent runs (dashboard tiles) never mixes one run's cap with + another's (fix-now F8, read-back C1). + + `page` records what Egeria actually sent, before any method collapses it into "No elements found": + "null" (the key absent or null: Egeria's "no more"), "empty" (`[]`: filtering emptied this page, so a + pager carries on) or "rows". The return value is unchanged by recording (pager step 2).""" + raw = response.json() + elements = raw.get(key, default) record = LIST_REQUEST_RECORD.get() if record is None: return elements + sent_value = raw.get(key) + record["page"] = ("null" if sent_value is None + else "empty" if isinstance(sent_value, list) and not sent_value else "rows") try: sent = json.loads(response.request.content or b"{}") except Exception: # noqa: BLE001 -- no request body (a GET) or not JSON: nothing to record diff --git a/pyegeria/core/config.py b/pyegeria/core/config.py index 1995bc12..0f9a1d92 100644 --- a/pyegeria/core/config.py +++ b/pyegeria/core/config.py @@ -548,6 +548,16 @@ def __getattr__(self, name): settings = _LazySettings() +def configured_user_id() -> str: + """The configured Egeria user, resolved now: EGERIA_USER as it is now, then the User Profile from pyegeria's + settings (config.json / .env), then the quickstart default. The one resolution of "the configured user": + `resolve_egeria_credentials` (which builds a CLI's client) and the report-spec cache's client-less identity + (`egeria_spec_cache.current_spec_identity`) both use it, so a CLI with no configuration binds its private + spec cache to, and loads it as, the same user. (pyegeria 7 drops the quickstart default; both sides then + become None together.)""" + return os.environ.get("EGERIA_USER") or settings.User_Profile.user_name or "erinoverview" + + def resolve_egeria_credentials(user_id: Optional[str] = None, user_pwd: Optional[str] = None) -> tuple[str, str]: """Resolve the Egeria user and password at call time (ISSUE-161). @@ -556,7 +566,7 @@ def resolve_egeria_credentials(user_id: Optional[str] = None, user_pwd: Optional .env), then the quickstart defaults. Call it when a command runs, not at import, so a variable set after import is still honoured. """ - user = user_id or os.environ.get("EGERIA_USER") or settings.User_Profile.user_name or "erinoverview" + user = user_id or configured_user_id() pwd = user_pwd or os.environ.get("EGERIA_USER_PASSWORD") or settings.User_Profile.user_pwd or "secret" return user, pwd diff --git a/pyegeria/core/mcp_server.py b/pyegeria/core/mcp_server.py index 0ad82c32..092ccbd3 100644 --- a/pyegeria/core/mcp_server.py +++ b/pyegeria/core/mcp_server.py @@ -13,7 +13,6 @@ from pyegeria.egeria_tech_client import EgeriaTech from pyegeria.core._exceptions import print_validation_error -from pyegeria.view.base_report_formats import load_egeria_report_specs GLOBAL_EGERIA_CLIENT: Optional[EgeriaTech] = None @@ -40,32 +39,67 @@ def _ok(result: Dict[str, Any]) -> Dict[str, Any]: return result +def _startup_client(settings): + """The server's startup Egeria client, signed in as the CONFIGURED identity, or None. + + The identity comes from pyegeria's settings (the config file, or EGERIA_USER / EGERIA_USER_PASSWORD), + the same as every other entry point. Nothing here names a user or a password: until 2026-11 this function's + body signed in with a hard-coded demo persona whatever the settings said, which put a credential in released + source and tied every MCP server to one persona. + + When no user or password is configured, the server starts WITHOUT Egeria and says so: the startup spec sync + is skipped, and the tools that read Egeria build their own client per call from the same settings, so they + report the missing identity themselves. + """ + user_id = settings.User_Profile.user_name + user_pwd = settings.User_Profile.user_pwd + if not user_id or not user_pwd: + logger.warning("pyegeria MCP server: no Egeria user or password is configured (EGERIA_USER / " + "EGERIA_USER_PASSWORD, or the pyegeria config file); starting without an Egeria " + "connection, and the startup report-spec sync is skipped.") + return None + logger.debug("Initializing Egeria client...") + client = EgeriaTech( + settings.Environment.egeria_view_server, + settings.Environment.egeria_view_server_url, + user_id, + user_pwd, + ) + logger.debug("Egeria Client initialized") + client.create_egeria_bearer_token(user_id, user_pwd) + logger.debug("Egeria Client connected") + return client + + def main() -> None: # Initialize the server global GLOBAL_EGERIA_CLIENT + # This server acts for every MCP caller (tools run as the credential the caller passes), so it declares + # itself multi-user: a read outside a caller's own identity takes no fallback identity (its startup client's + # included), and sees only the shared spec cache it syncs below (spec-cache R-A). + from pyegeria.core._client_identities import declare_multi_user + declare_multi_user() + try: """ Runs ONCE when the server starts. Initializes the Egeria client and stores it in the server's state. """ - logger.debug("Initializing Egeria client...") from pyegeria.core.config import settings as _settings - user_id = _settings.User_Profile.user_name - user_pwd = _settings.User_Profile.user_pwd - - GLOBAL_EGERIA_CLIENT = EgeriaTech( - _settings.Environment.egeria_view_server, - _settings.Environment.egeria_view_server_url, - user_id, - user_pwd - ) - logger.debug("Egeria Client initialized") - GLOBAL_EGERIA_CLIENT.create_egeria_bearer_token("erinoverview", "secret") - logger.debug("Egeria Client connected") - - load_egeria_report_specs(GLOBAL_EGERIA_CLIENT) - logger.debug("Egeria report specs loaded into registry") + GLOBAL_EGERIA_CLIENT = _startup_client(_settings) + if GLOBAL_EGERIA_CLIENT is not None: + # The spec cache's sync (the system of record for Egeria-held report specs) replaces the deprecated + # legacy ReportType:: loader, removed in pyegeria 7. A failed sync never stops the server. + try: + from pyegeria.view.egeria_spec_cache import sync_report_specs + # shared=True: this server's registry serves every caller, so its cache must never hold a zoned + # spec (egeria_spec_cache §9). An unshared cache would cache what the startup identity can see and + # serve it to every user of this server (the Portal-3 hazard). + report = sync_report_specs(GLOBAL_EGERIA_CLIENT, shared=True) + logger.debug(f"Egeria report specs synced into the spec cache: {report.render()}") + except Exception as sync_error: # noqa: BLE001 -- best effort at startup + logger.warning(f"report spec sync at startup failed ({type(sync_error).__name__}): {sync_error}") except ValidationError as e: print_validation_error(e) diff --git a/pyegeria/core/paging.py b/pyegeria/core/paging.py new file mode 100644 index 00000000..612ca8d7 --- /dev/null +++ b/pyegeria/core/paging.py @@ -0,0 +1,650 @@ +"""One pager for Egeria find/get calls: an empty page is not the end, and on some paths neither is null. + +Egeria's paging contract (finding-metadata/overview, "Paging", restated by the Egeria lead on 2026-11-13): +page with ``startFrom`` += ``pageSize`` until the server returns *null* (the ``elements`` key absent). An +*empty list* means server-side filtering (security, zones, Memento/lineage) removed every row of that page, +so the caller carries on. ``startFrom`` is an offset into the repository's rows *before* filtering, so the +pager always advances by the page size it asked for, never by the number of rows it got back. + +On some paths Egeria also sends null for a page that filtering emptied (metadata-expert by-search-conditions: +raw capture 2026-11-13, ``~/.re-gate/B5-RAW-CAPTURE-2026-11-13.md``; MetadataElementHandler returns null for +an empty filtered list). There, null is not proof of the end. A call site says which kind of path it pages +with ``end_policy``: + +- ``"trusted"``: null is the end (``end="confirmed"``). +- ``"suspect"`` (the default): on null, probe the next ``probe_pages`` pages. Rows there mean the null was a + filtered page and paging resumes; nothing there ends the read with ``end="not_confirmed"``. + +The result says how the read ended, so a caller never shows "nothing" or a total it didn't establish: + +- ``confirmed``: the server said there is no more, on a path where that can be trusted. +- ``not_confirmed``: the read stopped without proof of the end (a suspect null and empty probes, or too + many consecutive filtered positions). +- ``capped``: the caller's limits stopped it (``want`` rows collected, or ``max_pages``); there may be more. +- ``stalled``: a page brought only rows already read (a server ignoring ``startFrom``); stopped so it + can't loop forever. +- ``first_page``: one page was read and not paged on (a caller's choice, e.g. a report run whose first + page had rows). Set by callers, never by ``page_all``. + +Anything but ``confirmed`` is "at least N" (``PageResult.count_text``), never a total. + +``fetch(start, size)`` returns a list (rows; ``[]`` for a filtered page) or anything else (None, pyegeria's +"No elements found" string) for null. When ``fetch`` is a pyegeria list method, the pager reads what Egeria +actually sent from the method's list-request record, so a method that collapses ``[]`` into "No elements +found" still reads as a filtered page (``_end_signal``). Exceptions from ``fetch`` propagate: the caller decides what a failed +read means. This module only pages; it changes nothing about what pyegeria's public methods return. +""" +from __future__ import annotations + +import json +from contextlib import contextmanager +from dataclasses import dataclass, field, replace +from typing import Any, Awaitable, Callable, Hashable, List, Optional, Tuple + +DEFAULT_FETCH_PAGE_SIZE = 100 # rows asked for per request; below the server's maximum page size +DEFAULT_PROBE_PAGES = 3 # extra pages read after a null on a suspect path +DEFAULT_MAX_EMPTY_ROWS = 2000 # consecutive filtered positions before giving up (rows, not pages) +DEFAULT_MAX_PAGES = 1000 # requests per read, probes included + +END_POLICIES = ("trusted", "suspect") +ENDS = ("confirmed", "not_confirmed", "capped", "stalled", "first_page") +#: Why a read ended (PageResult.end_reason). Only the first two may be acted on as the end (reached_end). +END_REASONS = ("trusted_null", "null_then_empty_probes", "empty_run_cap", "want", "max_pages", "stalled") + +#: Methods whose null has been shown to mean the end (a filtered page comes back as []), with the evidence. +#: Everything else pages as "suspect". Add a method only with a raw capture like the one cited. +TRUSTED_END_METHODS = { + # content-status search: [] for 11 filtered pages, then 79 + 7 rows, then null (Architect, laz, 2026-11-13) + "find_authored_elements", +} + + +#: Methods proven to send null MID-SET (rows follow a null; a filtered-empty page is null, never []), so even a +#: null followed by empty probes is NOT an end on them: PageResult.reached_end is False, as for a limit, and no +#: write path may act on a read from them. Evidence: ~/.re-gate/CAPTURE-PAGER-STEP4-FINDINGS-2026-11-14.md +#: (laz, read-only, 2026-11-14): at pageSize 25, runs of up to 3 nulls mid-set on both, and no [] in 457 +#: requests. Promotion to TRUSTED_END_METHODS is closed for these until Egeria returns [] for a filtered page +#: (the Architect's ruling after the step 4 capture). +#: The by-property-value reads joined on the exists-lookups ruling (2026-11-14): classification-explorer +#: by-property-value is StewardshipManagementHandler -> OpenMetadataClient.getMetadataElementsByPropertyValue, +#: which is findMetadataElements(...getSearchPropertiesByName...): the server method the capture caught. +#: Promotion of any of these is gated on an Egeria build with odpi/egeria #9453 and a post-rebuild capture. +#: The by-name reads joined on the MyEgeria read-back (R3, 2026-11-14), traced read-only in Egeria 2a27dad5: +#: CollectionManager / CommunityMatters / FeedbackManager ".../by-name" REST -> CollectionHandler.getCollectionsByName +#: / CommunityHandler.getCommunitiesByName / NoteLogHandler.getNoteLogsByName -> OpenMetadataHandlerBase +#: .getRootElementsByName -> openMetadataClient.findMetadataElements(...getSearchPropertiesByName...): the same server +#: method as find_metadata_elements. (Their search-string twins go findRootElements -> findMetadataElementsWithString +#: -> OMF findElements -> handler.findEntities, and attached/member reads go getRelatedRootElements -> +#: getRelatedMetadataElements; those stay plain suspect.) +BY_NAME_METHODS = frozenset({"get_collections_by_name", "_async_get_collections_by_name", + "get_communities_by_name", "_async_get_communities_by_name", + "get_note_logs_by_name", "_async_get_note_logs_by_name"}) +#: get_elements (classification-explorer elements/by-type) -> ClassificationExplorerRESTServices.getRootElementsByType +#: -> StewardshipManagementHandler.getRootElementsByType -> openMetadataClient.findMetadataElements: the same server +#: method again (traced 2026-11-14 for P3 used_by, the Architect's ruling (c)), so NULL_NOT_END and gated like it. +BY_TYPE_METHODS = frozenset({"get_elements", "_async_get_elements"}) +NULL_NOT_END_METHODS = frozenset({"find_metadata_elements", "get_relationships", + "get_elements_by_property_value", "find_elements_by_property_value", + "_async_get_elements_by_property_value"}) | BY_NAME_METHODS | BY_TYPE_METHODS + +#: Probe pages after a null on a NULL_NOT_END method: the capture's longest null run was 3; 6 gives 2x headroom. +NULL_NOT_END_PROBE_PAGES = 6 + +assert not (TRUSTED_END_METHODS & NULL_NOT_END_METHODS), "a NULL_NOT_END method can't be trusted" + + +def end_policy_for(method_name: Optional[str]) -> str: + """"trusted" for a method in TRUSTED_END_METHODS (never one in NULL_NOT_END_METHODS), else "suspect".""" + if method_name in NULL_NOT_END_METHODS: + return "suspect" + return "trusted" if method_name in TRUSTED_END_METHODS else "suspect" + + +# ── Step 6: the #9453 version gate (the Architect's step 6 ruling, 2026-11-14) ───────────────────────────────── +# odpi/egeria #9453 (2a27dad5, merged 2026-11-14T19:52Z) fixed the find paths: on a platform with it, +# find_metadata_elements and by-property-value return [] for a filtered page and null only at the end (post-#9453 +# capture on laz, 5deda38d: ~/.re-gate/RESULTS-EGERIA-UPGRADE-5deda38d-AND-POST-9453-CAPTURE-2026-11-14.md). +# get_relationships by type is NOT fixed there (nulls mid-set, rows after, never []): it stays NULL_NOT_END on +# every platform until its own upstream fix and a capture. + +#: Promoted to trusted on a gated platform only. +GATED_TRUSTED_METHODS = frozenset({"find_metadata_elements", "get_elements_by_property_value", + "find_elements_by_property_value", "_async_get_elements_by_property_value"} + ) | BY_NAME_METHODS | BY_TYPE_METHODS # findMetadataElements underneath: #9453 +assert GATED_TRUSTED_METHODS < NULL_NOT_END_METHODS and "get_relationships" not in GATED_TRUSTED_METHODS + +GATE_SNAPSHOT_VERSION = "6.3-SNAPSHOT" +GATE_SNAPSHOT_TIME = "2026-11-14T19:52:00Z" # the #9453 merge +GATE_TTL_SECONDS = 3600 # a parsed /api/about answer +GATE_FAILED_TTL_SECONDS = 60 # a failed about-read: retried soon, so a platform restart doesn't turn trust off +GATE_ABOUT_TIMEOUT_SECONDS = 5 # the about-read's own timeout, where the client takes one +GATE_OVERRIDE_ENV = "PYEGERIA_TRUSTED_END" # "off" forces not-promoted everywhere; there is no "force on" + +_GATE_CACHE: dict = {} # platform URL -> (expires at, promoted) + + +def gate_from_about(about: Any) -> bool: + """Whether a platform's /api/about answer shows the #9453 paging fix. Fail-safe: anything not clearly + promoted (an older version or build time, unparsable, no time) is False. + + - promoted: a release later than 6.3-SNAPSHOT (6.3 or newer, not a SNAPSHOT), or "6.3-SNAPSHOT" built at or + after GATE_SNAPSHOT_TIME; + - any SNAPSHOT whose (major, minor) is later than 6.3 (e.g. 6.4-SNAPSHOT) is promoted: #9453 merged on main + before the 6.3 line was cut (step 6 read-back R1); an earlier SNAPSHOT (6.2-SNAPSHOT) is not; + - known limit: a 6.3-SNAPSHOT built after that time from code without #9453 reads as promoted + (PYEGERIA_TRUSTED_END=off covers it).""" + import re + from datetime import datetime + try: + if isinstance(about, (str, bytes)): + about = json.loads(about) + if not isinstance(about, dict): + return False + version = str(about.get("version") or "").strip() + m = re.fullmatch(r"(\d+)\.(\d+)(?:\.(\d+))?(-SNAPSHOT)?", version) + if not m: + return False + numbers = tuple(int(g or 0) for g in m.groups()[:3]) + if not m.group(4): + return numbers >= (6, 3, 0) + if numbers[:2] > (6, 3): # step 6 read-back R1: every SNAPSHOT after 6.3 contains #9453 (merged on main) + return True + if version != GATE_SNAPSHOT_VERSION: + return False + built = about.get("time") + if not built: + return False + when = datetime.fromisoformat(str(built).replace("Z", "+00:00")) + gate = datetime.fromisoformat(GATE_SNAPSHOT_TIME.replace("Z", "+00:00")) + return when.tzinfo is not None and when >= gate + except Exception: # noqa: BLE001 -- fail safe: never trusted on unknown + return False + + +def _gate_overridden() -> bool: + import os + return os.environ.get(GATE_OVERRIDE_ENV, "").strip().lower() == "off" + + +def _gate_cached(platform_url: str) -> Optional[bool]: + import time + hit = _GATE_CACHE.get(platform_url) + if hit is not None and hit[0] > time.monotonic(): + return hit[1] + return None + + +def _gate_store(platform_url: str, promoted: bool, failed: bool = False) -> bool: + import time + ttl = GATE_FAILED_TTL_SECONDS if failed else GATE_TTL_SECONDS + _GATE_CACHE[platform_url] = (time.monotonic() + ttl, promoted) + return promoted + + +def _about_reader(client: Any, name: str): + """The client's own platform-origin reader (``name``), or a subclient's (EgeriaTech proxies), or None.""" + reader = getattr(type(client), name, None) and getattr(client, name, None) + if reader is None: + for sub in ("classification_manager", "metadata_expert"): + inner = getattr(client, sub, None) + reader = getattr(inner, name, None) if inner is not None else None + if reader is not None: + break + return reader + + +def _call_about(reader: Callable[[], Any]) -> Any: + """The sync about-read, bounded by GATE_ABOUT_TIMEOUT_SECONDS (on a worker thread, which is abandoned if + it overruns: the read is only ever "not promoted" then).""" + from concurrent.futures import ThreadPoolExecutor + pool = ThreadPoolExecutor(max_workers=1, thread_name_prefix="pyegeria-about") + try: + return pool.submit(_read_and_close, reader).result(timeout=GATE_ABOUT_TIMEOUT_SECONDS) + finally: + pool.shutdown(wait=False) + + +def _read_and_close(reader: Callable[[], Any]) -> Any: + """Run the about-read on this worker thread, then close what the read opened here: the client's + thread-local session and this thread's run_sync loop (step 6 gate read-back advisory: one was left unclosed + per platform per TTL).""" + try: + return reader() + finally: + _close_thread_resources(getattr(reader, "__self__", None)) + + +def _close_thread_resources(owner: Any) -> None: + from pyegeria.core import async_runner + try: + local = getattr(owner, "_thread_local", None) + session = getattr(local, "session", None) if local is not None else None + loop = getattr(async_runner._thread_state, "loop", None) + if session is not None and loop is not None and not loop.is_closed(): + loop.run_until_complete(session.aclose()) + if local is not None: + for name in ("session", "session_loop"): + if hasattr(local, name): + delattr(local, name) + if loop is not None and not loop.is_closed(): + loop.close() + except Exception: # noqa: BLE001 -- cleanup only: never turns a read into an error + pass + + +def platform_promoted(client: Any) -> bool: + """Whether `client`'s platform carries the #9453 fix: one best-effort /api/about read per platform URL per + GATE_TTL_SECONDS. Never raises, never blocks a read: a failed or unparsable read is "not promoted".""" + if client is None or _gate_overridden(): + return False + url = getattr(client, "platform_url", None) + if not isinstance(url, str) or not url: + return False + if (cached := _gate_cached(url)) is not None: + return cached + try: + reader = _about_reader(client, "get_platform_origin") + about = _call_about(reader) if reader is not None else None + except Exception: # noqa: BLE001 -- a failed about-read only means "not promoted" + about = None + return _gate_store(url, gate_from_about(about), failed=not about) + + +async def async_platform_promoted(client: Any) -> bool: + """``platform_promoted`` for async code (the same cache).""" + if client is None or _gate_overridden(): + return False + url = getattr(client, "platform_url", None) + if not isinstance(url, str) or not url: + return False + if (cached := _gate_cached(url)) is not None: + return cached + try: + reader = _about_reader(client, "async_get_platform_origin") + about = None + if reader is not None: + import asyncio + about = await asyncio.wait_for(reader(), timeout=GATE_ABOUT_TIMEOUT_SECONDS) + except Exception: # noqa: BLE001 -- a failed or slow about-read only means "not promoted" + about = None + return _gate_store(url, gate_from_about(about), failed=not about) + + +def _paging(method_name: Optional[str], promoted: bool) -> dict: + if promoted and method_name in GATED_TRUSTED_METHODS: + return {"end_policy": "trusted"} + if method_name in NULL_NOT_END_METHODS: + return {"end_policy": "suspect", "probe_pages": NULL_NOT_END_PROBE_PAGES, "null_not_end": True} + return {"end_policy": end_policy_for(method_name)} + + +def paging_for(method_name: Optional[str], client: Any = None) -> dict: + """page_all keyword arguments for a method: its end_policy and, for a NULL_NOT_END method, the longer + probe and ``null_not_end`` (so a null-then-empty-probes read is never ``reached_end``). + + Given the ``client`` that makes the read, a GATED_TRUSTED_METHODS method is trusted on a platform with the + #9453 fix (``platform_promoted``). The platform is only asked when the method is gated.""" + promoted = method_name in GATED_TRUSTED_METHODS and platform_promoted(client) + return _paging(method_name, promoted) + + +async def async_paging_for(method_name: Optional[str], client: Any = None) -> dict: + """``paging_for`` for async code: the platform's /api/about is read without blocking the event loop.""" + promoted = method_name in GATED_TRUSTED_METHODS and await async_platform_promoted(client) + return _paging(method_name, promoted) + + +@dataclass(frozen=True) +class PageResult: + """What a paged read found and how it ended. ``items`` is cut to ``want`` when one was given.""" + items: List[Any] + end: str + note: Optional[str] = None + pages_read: int = 0 + positions_read: int = 0 # how far into the server's (pre-filter) rows the read got + end_reason: Optional[str] = None # why it ended: one of END_REASONS (None only for a caller-built result) + null_not_end: bool = False # read from a NULL_NOT_END method: a null is never the end + + @property + def complete(self) -> bool: + """True only when the server confirmed there is nothing more.""" + return self.end == "confirmed" + + @property + def reached_end(self) -> bool: + """True when the read may be acted on as the end (step-4 ruling (A)): a trusted null, or a suspect + null whose probes found nothing. ``complete`` is stricter: only it allows a total or "none" to be + shown. A limit (capped, stalled, the empty-run cap) is never the end.""" + if self.end_reason == "null_then_empty_probes" and self.null_not_end: + return False # NULL_NOT_END_METHODS: rows have followed a null on this method + return self.end_reason in ("trusted_null", "null_then_empty_probes") + + def trace(self) -> dict: + """How the read ended, for a log line or a run result: positions_read, end, end_reason.""" + return {"positions_read": self.positions_read, "end": self.end, "end_reason": self.end_reason} + + def count_text(self) -> str: + """The count as it may be shown: "N" when complete, else "at least N".""" + n = len(self.items) + return str(n) if self.complete else f"at least {n}" + + +_GUID_FIELDS = ("guid", "elementGUID", "relationshipGUID") + + +def default_key(item: Any) -> Hashable: + """A row's GUID, whichever shape it comes in: + + - an Egeria 6 element or relationship: ``elementHeader.guid`` / ``relationshipHeader.guid``; + - the metadata-expert shape (``OpenMetadataElement``, ``OpenMetadataRelationship``): ``elementGUID`` / + ``relationshipGUID`` on the row itself; + - a bare ``guid`` on the row. + + A row with none of these is keyed by its content (equal JSON, equal key), never by object identity: + every page is decoded afresh, so the same row read twice is two different objects.""" + if isinstance(item, dict): + for holder in (item.get("elementHeader"), item.get("relationshipHeader"), item): + if isinstance(holder, dict): + for name in _GUID_FIELDS: + if isinstance(holder.get(name), str) and holder[name]: + return holder[name] + try: + return "content:" + json.dumps(item, sort_keys=True, default=str) + except (TypeError, ValueError): # a circular or otherwise undumpable row: identity is all there is + return id(item) + + +@dataclass +class _Pager: + """The paging state machine, shared by the sync and async drivers. ``request()`` gives the next + (start, size) to fetch or None when done; ``feed(page)`` takes what that fetch returned.""" + want: Optional[int] + size: int + policy: str + probe_pages: int + max_empty_rows: int + max_pages: int + key: Callable[[Any], Hashable] + start: int = 0 + stall_pages: int = 1 # consecutive pages of only rows already read before "stalled" + no_new: int = 0 + last_keys: Optional[list] = None # the previous page's row keys: the same page again is a stall + pages: int = 0 + empty_rows: int = 0 + probes_left: Optional[int] = None # None: not probing + null_at: Optional[int] = None # where the suspect null came back + items: List[Any] = field(default_factory=list) + seen: set = field(default_factory=set) + result: Optional[PageResult] = None + + def request(self) -> Optional[Tuple[int, int]]: + if self.result is not None: + return None + if self.pages >= self.max_pages: + self._finish("capped", f"stopped after {self.pages} pages of {self.size}; there may be more", + "max_pages") + return None + return self.start, self.size + + def feed(self, page: Any) -> None: + self.pages += 1 + at = self.start + self.start += self.size + if not isinstance(page, list): + self._null(at) + elif not page: + self._empty(at) + else: + self._rows(page, at) + + # ── one handler per kind of page ── + + def _null(self, at: int) -> None: + if self.policy == "trusted": + self._finish("confirmed", None, "trusted_null") + return + if self.probes_left is None: + self.probes_left, self.null_at = self.probe_pages, at + else: + self.probes_left -= 1 + if self.probes_left <= 0: + self._finish("not_confirmed", + f"Egeria reported no more at position {self.null_at}, and the next " + f"{self.probe_pages} pages of {self.size} were empty too; on this path a filtered page " + f"also reads as 'no more', so later matches can't be ruled out", "null_then_empty_probes") + + def _empty(self, at: int) -> None: + self.probes_left = None # an empty page means the server has rows here: not the end + self.empty_rows += self.size + if self.empty_rows >= self.max_empty_rows: + self._finish("not_confirmed", + f"{self.empty_rows} consecutive positions (up to {at + self.size}) were filtered out " + f"for you; there may be more", "empty_run_cap") + + def _rows(self, page: list, at: int) -> None: + self.probes_left = None + self.empty_rows = 0 + fresh = [] + keys = [] + for item in page: + k = self.key(item) + keys.append(k) + if k not in self.seen: + self.seen.add(k) + fresh.append(item) + same_as_last, self.last_keys = keys == self.last_keys, keys + if not fresh: + self.no_new += 1 + # overlapping short pages (export_dcat) are not yet a stall; the very same page again always is + if self.no_new < self.stall_pages and not same_as_last: + return + self._finish("stalled", + f"the read from position {at} returned only rows already read (the server may be " + f"ignoring startFrom), so it stopped at {len(self.items)}", "stalled") + return + self.no_new = 0 + self.items.extend(fresh) + if self.want is not None and len(self.items) >= self.want: + self._finish("capped", f"stopped at the {self.want} rows asked for; there may be more", "want") + + def _finish(self, end: str, note: Optional[str], reason: str) -> None: + items = self.items[: self.want] if self.want is not None else self.items + self.result = PageResult(items=list(items), end=end, note=note, pages_read=self.pages, + positions_read=self.start, end_reason=reason) + + +def _pager(want, fetch_page_size, end_policy, probe_pages, max_empty_rows, max_pages, key, + start_from=0, stall_pages=1) -> _Pager: + if end_policy not in END_POLICIES: + raise ValueError(f"end_policy must be one of {END_POLICIES}, not {end_policy!r}") + for name, value, low in (("fetch_page_size", fetch_page_size, 1), ("probe_pages", probe_pages, 1), + ("max_empty_rows", max_empty_rows, 1), ("max_pages", max_pages, 1), + ("stall_pages", stall_pages, 1)): + if not isinstance(value, int) or isinstance(value, bool) or value < low: + raise ValueError(f"{name} must be a whole number of at least {low}, not {value!r}") + if want is not None and (not isinstance(want, int) or isinstance(want, bool) or want < 1): + raise ValueError(f"want must be None or a whole number of at least 1, not {want!r}") + if not isinstance(start_from, int) or isinstance(start_from, bool) or start_from < 0: + raise ValueError(f"start_from must be a whole number of at least 0, not {start_from!r}") + return _Pager(want=want, size=fetch_page_size, policy=end_policy, probe_pages=probe_pages, + max_empty_rows=max_empty_rows, max_pages=max_pages, key=key or default_key, start=start_from, + stall_pages=stall_pages) + + +@contextmanager +def _end_signal(): + """Record what Egeria actually sent for the list request ``fetch`` makes (pager step 2). + + pyegeria's list helpers note it in ``LIST_REQUEST_RECORD["page"]`` ("null", "empty" or "rows") before a + method collapses ``[]`` into "No elements found", so the pager can tell a filtered page from the end + without any public method changing what it returns. A caller already recording (``exec_report_spec`` + reads ``cap``) gets this request's facts merged into its own record afterwards, as if this block were + not there: the record keeps the last list request, as it always has.""" + from pyegeria.core._server_client import LIST_REQUEST_RECORD, recording_list_requests + outer = LIST_REQUEST_RECORD.get() + with recording_list_requests() as record: + yield record + if outer is not None: + outer.update(record) + + +def _signalled(page: Any, record: dict) -> Any: + """``page`` as the pager should read it: ``[]`` when Egeria sent an empty list that the method collapsed, + None when Egeria sent null. When no list helper recorded anything (a fetch that isn't a pyegeria list + call, or a path not yet routed through the record), ``page`` is read as it is.""" + sent = record.get("page") + if sent == "empty" and not (isinstance(page, list) and page): + return [] + if sent == "null" and not isinstance(page, list): + return None + return page + + +def page_all(fetch: Callable[[int, int], Any], *, want: Optional[int] = None, + fetch_page_size: int = DEFAULT_FETCH_PAGE_SIZE, end_policy: str = "suspect", + probe_pages: int = DEFAULT_PROBE_PAGES, max_empty_rows: int = DEFAULT_MAX_EMPTY_ROWS, + max_pages: int = DEFAULT_MAX_PAGES, key: Optional[Callable[[Any], Hashable]] = None, + start_from: int = 0, stall_pages: int = 1, null_not_end: bool = False) -> PageResult: + """Read every row ``fetch(start, size)`` gives (or the first ``want``), paging until the end is known + or a limit stops it, starting at ``start_from``. See the module docstring for ``end_policy`` and the + ``end`` states. + + ``key`` gives each row's identity, for dropping rows read twice and for the stall guard. The default + (``default_key``) is the row's GUID in any of Egeria's shapes, else its JSON content. A caller paging + rows that have no GUID and can legitimately repeat (aggregates, counts, value lists) must pass its own + ``key``: under the content fallback two equal rows are one row, and a page of them reads as a stall. + + Each ``fetch`` runs inside ``_end_signal``, so a pyegeria list method that collapses ``[]`` into + "No elements found" is still read as a filtered page, not as the end. + + ``stall_pages`` is how many pages in a row bringing only rows already read end the read as "stalled" + (default 1; a path known to return overlapping short pages may allow 2). + + ``null_not_end`` (a NULL_NOT_END method: pass ``**paging_for(method)``) marks the result so that a null + followed by empty probes is not ``reached_end``.""" + pager = _pager(want, fetch_page_size, end_policy, probe_pages, max_empty_rows, max_pages, key, start_from, + stall_pages) + while (req := pager.request()) is not None: + with _end_signal() as record: + page = fetch(*req) + pager.feed(_signalled(page, record)) + return replace(pager.result, null_not_end=True) if null_not_end else pager.result + + +async def async_page_all(fetch: Callable[[int, int], Awaitable[Any]], *, want: Optional[int] = None, + fetch_page_size: int = DEFAULT_FETCH_PAGE_SIZE, end_policy: str = "suspect", + probe_pages: int = DEFAULT_PROBE_PAGES, max_empty_rows: int = DEFAULT_MAX_EMPTY_ROWS, + max_pages: int = DEFAULT_MAX_PAGES, + key: Optional[Callable[[Any], Hashable]] = None, start_from: int = 0, + stall_pages: int = 1, null_not_end: bool = False) -> PageResult: + """``page_all`` for an async ``fetch`` (same ``key`` and end-signal rules).""" + pager = _pager(want, fetch_page_size, end_policy, probe_pages, max_empty_rows, max_pages, key, start_from, + stall_pages) + while (req := pager.request()) is not None: + with _end_signal() as record: + page = await fetch(*req) + pager.feed(_signalled(page, record)) + return replace(pager.result, null_not_end=True) if null_not_end else pager.result + + +# ── Step 4: what a loop says and logs about how its read ended (ruling (A), 2026-11-13) ── + +def at_least_note(paged: PageResult, what: str, count: Optional[int] = None) -> Optional[str]: + """The words a loop that shows a count or a list adds when the end isn't confirmed, or None when it is: + "Showing at least N : ." Never a total, never "none". ``count`` is what the loop shows when it + filters the rows it read (default: the rows read).""" + if paged.complete: + return None + n = len(paged.items) if count is None else count + return f"Showing at least {n} {what}: {paged.note or 'there may be more'}." + + +def log_end(method: str, paged: PageResult) -> dict: + """Log one structured line for a read (method, positions_read, end, end_reason) and return those fields, + so a write path can carry them in its own result and a later miss can be traced to the read.""" + from loguru import logger + fields = {"method": method, **paged.trace()} + if paged.complete: + logger.debug(f"paged read: {fields}") + else: + logger.info(f"paged read: {fields}") + return fields + + +# ── Step 5: "exists?" lookups (the exists-lookups ruling, 2026-11-14) ───────────────────────────────────────── +# found = a match was read; absent = the read reached the end (never on a NULL_NOT_END method); unknown otherwise. +# Use only where one "found" decides the outcome, never to prove uniqueness. A write guard treats unknown as +# "don't create; log it". ("None" as display words still needs a confirmed end: brief condition 4.) + +EXISTS_MAX_PAGES = 10 +EXISTS_ANSWERS = ("found", "absent", "unknown") + + +@dataclass(frozen=True) +class Exists: + """A three-way existence answer, with the matches read and how the read ended.""" + answer: str + matches: List[Any] + read: PageResult + + @property + def found(self) -> bool: + return self.answer == "found" + + def words(self, what: str) -> str: + if self.answer == "found": + return f"{what}: found" + if self.answer == "absent": + return f"{what}: not found (the read reached the end)" + return (f"{what}: can't tell whether it exists ({self.read.positions_read} positions read, end " + f"{self.read.end}, {self.read.end_reason}): Egeria didn't confirm there are no more") + + +def _exists_pager(method: Optional[str], fetch_page_size: int, max_pages: int, p: dict) -> _Pager: + if not isinstance(max_pages, int) or isinstance(max_pages, bool) or not 1 <= max_pages <= EXISTS_MAX_PAGES: + raise ValueError(f"max_pages must be a whole number from 1 to {EXISTS_MAX_PAGES}, not {max_pages!r}") + return _pager(None, fetch_page_size, p["end_policy"], p.get("probe_pages", DEFAULT_PROBE_PAGES), + DEFAULT_MAX_EMPTY_ROWS, max_pages, None) + + +def _exists_answer(pager: _Pager, matches: List[Any], p: dict) -> Exists: + result = pager.result or PageResult(items=list(pager.items), end="capped", note="stopped at the first match", + pages_read=pager.pages, positions_read=pager.start, end_reason="want") + if p.get("null_not_end"): + result = replace(result, null_not_end=True) + if matches: + return Exists("found", matches, result) + return Exists("absent" if result.reached_end else "unknown", [], result) + + +def exists(fetch: Callable[[int, int], Any], match: Callable[[Any], bool], *, method: Optional[str], + fetch_page_size: int = DEFAULT_FETCH_PAGE_SIZE, max_pages: int = EXISTS_MAX_PAGES, + client: Any = None) -> Exists: + """Page ``fetch`` (with ``**paging_for(method, client)``) until a row satisfies ``match`` (an exact, + client-side test), the end is reached, or ``max_pages`` (at most EXISTS_MAX_PAGES) stops it.""" + p = paging_for(method, client) + pager = _exists_pager(method, fetch_page_size, max_pages, p) + matches: List[Any] = [] + while not matches and (req := pager.request()) is not None: + with _end_signal() as record: + page = fetch(*req) + before = len(pager.items) + pager.feed(_signalled(page, record)) + matches = [row for row in pager.items[before:] if match(row)] + return _exists_answer(pager, matches, p) + + +async def async_exists(fetch: Callable[[int, int], Awaitable[Any]], match: Callable[[Any], bool], *, + method: Optional[str], fetch_page_size: int = DEFAULT_FETCH_PAGE_SIZE, + max_pages: int = EXISTS_MAX_PAGES, client: Any = None) -> Exists: + """``exists`` for an async ``fetch``.""" + p = await async_paging_for(method, client) + pager = _exists_pager(method, fetch_page_size, max_pages, p) + matches: List[Any] = [] + while not matches and (req := pager.request()) is not None: + with _end_signal() as record: + page = await fetch(*req) + before = len(pager.items) + pager.feed(_signalled(page, record)) + matches = [row for row in pager.items[before:] if match(row)] + return _exists_answer(pager, matches, p) diff --git a/pyegeria/core/request_filters.py b/pyegeria/core/request_filters.py index c3d555e6..9bd3c0b7 100644 --- a/pyegeria/core/request_filters.py +++ b/pyegeria/core/request_filters.py @@ -140,7 +140,7 @@ class ListFilters(GetFilters, total=False): anchor_scope_guid: str """Only return elements whose anchor scope is the element with this GUID.""" limit_results_by_status: list[str] - """Only return elements whose status is one of these, e.g. ``["ACTIVE", "DRAFT"]``.""" + """Only return elements whose status is one of these, e.g. ``["ACTIVE"]``; Egeria's ElementStatus: UNKNOWN, ACTIVE, INVALID, DELETED, OTHER (a content status such as DRAFT is refused).""" sequencing_order: Literal[ "ANY", "CREATION_DATE_RECENT", "CREATION_DATE_OLDEST", "LAST_UPDATE_RECENT", "LAST_UPDATE_OLDEST", "PROPERTY_ASCENDING", "PROPERTY_DESCENDING", diff --git a/pyegeria/core/unique_name.py b/pyegeria/core/unique_name.py new file mode 100644 index 00000000..5e701741 --- /dev/null +++ b/pyegeria/core/unique_name.py @@ -0,0 +1,210 @@ +""" +SPDX-License-Identifier: Apache-2.0 +Copyright Contributors to the ODPi Egeria project. + +"Which element has this qualified name?" answered by Egeria's exact unique-name read, three ways. + +A qualified name is unique in Egeria, and Egeria answers it server-side and exactly +(classification-explorer ``elements/by-unique-name`` -> getMetadataElementByUniqueName): no client paging, so +neither a filtered page nor the B5 null-mid-set can make it read as "absent". A paged ``get_*_by_name`` search +(findMetadataElements underneath, NULL_NOT_END before odpi/egeria #9453) must never decide create-versus-reuse +or "not found" for an exact qualified name (the exists-lookups / publish ruling; MyEgeria read-back R2). + +- found: the element (of ``type_name``, when given); +- absent: Egeria's specific not-found (``NO_ELEMENTS_FOUND`` or ``PyegeriaNotFoundException``), or an element of + another type holds the name (``wrong_type``: then no element of ``type_name`` can hold it). wrong_type -> + absent is right for READERS only: a write guard must REFUSE on wrong_type, because Egeria rejects a second + element with the same qualified name; +- unknown: any other answer or error. A write guard refuses on unknown and writes nothing; a reader says it + couldn't tell (``words``), never "not found". +""" +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Dict, List, Optional + +from pyegeria.core._exceptions import PyegeriaNotFoundException +from pyegeria.core._globals import NO_ELEMENTS_FOUND + +ANSWERS = ("found", "absent", "unknown") + + +@dataclass(frozen=True) +class UniqueName: + answer: str + qualified_name: str + element: Optional[Dict[str, Any]] = None + reason: Optional[str] = None + wrong_type: Optional[str] = None # absent because an element of this other type holds the name + + @property + def found(self) -> bool: + return self.answer == "found" + + @property + def guid(self) -> str: + return str(((self.element or {}).get("elementHeader") or {}).get("guid") or "") + + def words(self, what: str) -> str: + if self.answer == "found": + return f"{what} {self.qualified_name}: found" + if self.answer == "absent": + held = f" (a {self.wrong_type} holds that qualified name)" if self.wrong_type else "" + return f"{what} {self.qualified_name}: not found{held}" + return (f"Couldn't tell whether Egeria holds {what} {self.qualified_name}: " + f"{self.reason or 'no clear answer'}") + + +class UniqueNameUnknown(RuntimeError): + """Raised by readers that turn "unknown" into an error (never into "not found"); the message is the words.""" + + def __init__(self, found: UniqueName, what: str): + super().__init__(found.words(what)) + self.found = found + + +def type_names(element: Dict[str, Any]) -> List[str]: + t = (element.get("elementHeader") or {}).get("type") or {} + return [n for n in (t.get("typeName"), *(t.get("superTypeNames") or [])) if n] + + +def _element_of(result: Any) -> Optional[Dict[str, Any]]: + if isinstance(result, dict) and result.get("elementHeader"): + return result + if isinstance(result, dict) and isinstance(result.get("element"), dict): + return result["element"] + return None + + +def answer_for(result: Any, qualified_name: str, type_name: Optional[str] = None) -> UniqueName: + """Classify what the unique-name read returned.""" + if result == NO_ELEMENTS_FOUND: + return UniqueName("absent", qualified_name) + element = _element_of(result) + if element is None or not (element.get("elementHeader") or {}).get("guid"): + return UniqueName("unknown", qualified_name, reason="Egeria's answer wasn't an element or 'not found'") + if type_name and type_name not in type_names(element): + held = (type_names(element) or ["different element"])[0] + return UniqueName("absent", qualified_name, element, wrong_type=held) + return UniqueName("found", qualified_name, element) + + +def answer_for_error(exc: BaseException, qualified_name: str) -> UniqueName: + """Classify an error from the unique-name read: only Egeria's specific not-found is "absent".""" + if isinstance(exc, PyegeriaNotFoundException): + return UniqueName("absent", qualified_name) + try: + detail = " ".join(str(exc).split())[:200] + except Exception: # noqa: BLE001 -- an exception that can't say itself still has a class name + detail = "" + return UniqueName("unknown", qualified_name, + reason=f"the unique-name read failed: {type(exc).__name__}" + (f": {detail}" if detail else "")) + + +def _text(value: Any) -> Optional[str]: + return (value.strip() or None) if isinstance(value, str) else None + + +@dataclass +class _Read: + """How to make the unique-name read: `fn`, plus the explorer built for it (closed after), or `unknown`.""" + fn: Any = None + explorer: Any = None + unknown: Optional[str] = None + + +def _reader(client: Any, name: str) -> _Read: + """The unique-name read on `client`, its classification_manager, or a ClassificationExplorer made on the + same platform AS THE SAME CALLER (CollectionManager has no unique-name read of its own; READBACK U1): + + - the caller's token, built with ``user_pwd=NO_PASSWORD``: the explorer never holds the settings default + password, so a 401 auto-retry can't sign in again with it (C2); + - with no token, ``carried_password(client)``: the password the caller holds, or its NO_PASSWORD opt-out + (then, with no token either, unknown); never a settings default; + - with neither, or no real ``view_server``/``platform_url`` strings (a fake, a non-client): unknown, with no + client built and no call to Egeria. + """ + fn = getattr(client, name, None) + if callable(fn): + return _Read(fn) + sub = getattr(client, "classification_manager", None) + fn = getattr(sub, name, None) if sub is not None else None + if callable(fn): + return _Read(fn) + view_server = _text(getattr(client, "view_server", None)) + platform_url = _text(getattr(client, "platform_url", None)) + if not (view_server and platform_url): + return _Read(unknown="the caller isn't a client on an Egeria platform, so no unique-name read was made") + from pyegeria.core._helper_client import bare_token, helper_client, helper_password + from pyegeria.core._no_password import NO_PASSWORD + token = bare_token(client) + user_id = _text(getattr(client, "user_id", None)) + if not token: + password = helper_password(client, None) + if not (user_id and password is not NO_PASSWORD and _text(password)): + return _Read(unknown="the caller holds neither a token nor its own password, so no unique-name read " + "was made") + from pyegeria.omvs.classification_explorer import ClassificationExplorer + explorer = helper_client(ClassificationExplorer, client, view_server=view_server) # C3: the one helper + return _Read(getattr(explorer, name), explorer) + + +def _close(explorer: Any) -> None: + try: + explorer.close_session() + except Exception: # noqa: BLE001 -- closing is best-effort; the answer stands + pass + + +async def _async_close(explorer: Any) -> None: + try: + await explorer._async_close_session() + except Exception: # noqa: BLE001 -- closing is best-effort; the answer stands + pass + + +def by_unique_name(client: Any, qualified_name: str, type_name: Optional[str] = None) -> UniqueName: + """Egeria's exact unique-name read of `qualified_name` (optionally of `type_name` or a subtype). Never raises: + an error is "unknown" (or "absent" for Egeria's specific not-found).""" + try: + read = _reader(client, "get_element_by_unique_name") + except Exception as exc: # noqa: BLE001 -- couldn't even build the read: unknown, never absent + return UniqueName("unknown", qualified_name, reason=f"the unique-name read failed: {type(exc).__name__}") + if read.unknown: + return UniqueName("unknown", qualified_name, reason=read.unknown) + try: + result = read.fn(qualified_name) + except Exception as exc: # noqa: BLE001 -- only Egeria's specific not-found is "absent" + return answer_for_error(exc, qualified_name) + finally: + if read.explorer is not None: + _close(read.explorer) + return answer_for(result, qualified_name, type_name) + + +async def async_by_unique_name(client: Any, qualified_name: str, type_name: Optional[str] = None) -> UniqueName: + """``by_unique_name`` for async code.""" + try: + read = _reader(client, "_async_get_element_by_unique_name") + except Exception as exc: # noqa: BLE001 -- couldn't even build the read: unknown, never absent + return UniqueName("unknown", qualified_name, reason=f"the unique-name read failed: {type(exc).__name__}") + if read.unknown: + return UniqueName("unknown", qualified_name, reason=read.unknown) + try: + result = await read.fn(qualified_name) + except Exception as exc: # noqa: BLE001 -- only Egeria's specific not-found is "absent" + return answer_for_error(exc, qualified_name) + finally: + if read.explorer is not None: + await _async_close(read.explorer) + return answer_for(result, qualified_name, type_name) + + +def element_by_unique_name(client: Any, qualified_name: str, type_name: Optional[str] = None, + what: str = "the element") -> Optional[Dict[str, Any]]: + """For a reader: the element, None when Egeria says there is none (of `type_name`), and + UniqueNameUnknown (never None) when it can't tell.""" + found = by_unique_name(client, qualified_name, type_name) + if found.answer == "unknown": + raise UniqueNameUnknown(found, what) + return found.element if found.found else None diff --git a/pyegeria/egeria_client.py b/pyegeria/egeria_client.py index 1ab38743..09a438d6 100644 --- a/pyegeria/egeria_client.py +++ b/pyegeria/egeria_client.py @@ -12,6 +12,7 @@ """ +from pyegeria.core._no_password import NO_PASSWORD, carried_password, password_as_given, resolve_password from pyegeria.egeria_config_client import EgeriaConfig from pyegeria.omvs.action_author import ActionAuthor from pyegeria.omvs.actor_manager import ActorManager @@ -72,7 +73,7 @@ def __init__( self.view_server = view_server or settings.Environment.egeria_view_server self.platform_url = platform_url or settings.Environment.egeria_platform_url self.user_id = user_id or settings.User_Profile.user_name - self.user_pwd = user_pwd or settings.User_Profile.user_pwd + self.user_pwd, self.use_default_password = resolve_password(user_pwd) # NO_PASSWORD: opt out self.token = token # See BaseServerClient._token_minted_for: None means caller-supplied. self._token_minted_for = None @@ -142,17 +143,11 @@ def __init__( self._instantiated_clients = {} def _get_subclient(self, attr_name: str): + """Lazy-load and cache sub-clients, each built to act for this client (helper_client, C3).""" if attr_name not in self._instantiated_clients: - client_cls = self._subclient_map[attr_name] - self._instantiated_clients[attr_name] = client_cls( - self.view_server, - self.platform_url, - self.user_id, - self.user_pwd, - self.token, - timeout=self.timeout, - ) - self._instantiated_clients[attr_name]._token_minted_for = self._token_minted_for + from pyegeria.core._helper_client import helper_client + self._instantiated_clients[attr_name] = helper_client( + self._subclient_map[attr_name], self, timeout=self.timeout) return self._instantiated_clients[attr_name] def __getattr__(self, name): diff --git a/pyegeria/egeria_config_client.py b/pyegeria/egeria_config_client.py index 489347db..89688cf8 100644 --- a/pyegeria/egeria_config_client.py +++ b/pyegeria/egeria_config_client.py @@ -5,6 +5,7 @@ Client to configure the Egeria platform and servers """ +from pyegeria.core._no_password import NO_PASSWORD, carried_password, password_as_given, resolve_password from pyegeria.omvs.core_omag_server_config import CoreServerConfig from pyegeria.omvs.full_omag_server_config import FullServerConfig from pyegeria.omvs.server_operations import ServerOps @@ -28,7 +29,7 @@ def __init__( self.view_server = view_server or settings.Environment.egeria_view_server self.platform_url = platform_url or settings.Environment.egeria_platform_url self.user_id = user_id or settings.User_Profile.user_name - self.user_pwd = user_pwd or settings.User_Profile.user_pwd + self.user_pwd, self.use_default_password = resolve_password(user_pwd) # NO_PASSWORD: opt out self.token = token self.timeout = timeout @@ -39,16 +40,11 @@ def __init__( self._instantiated_clients = {} def _get_subclient(self, attr_name: str): + """Lazy-load and cache sub-clients, each built to act for this client (helper_client, C3).""" if attr_name not in self._instantiated_clients: - client_cls = self._subclient_map[attr_name] - self._instantiated_clients[attr_name] = client_cls( - self.view_server, - self.platform_url, - self.user_id, - self.user_pwd, - self.token, - timeout=self.timeout, - ) + from pyegeria.core._helper_client import helper_client + self._instantiated_clients[attr_name] = helper_client( + self._subclient_map[attr_name], self, timeout=self.timeout) return self._instantiated_clients[attr_name] def __getattr__(self, name): diff --git a/pyegeria/egeria_tech_client.py b/pyegeria/egeria_tech_client.py index b9b3ee4c..a508241d 100644 --- a/pyegeria/egeria_tech_client.py +++ b/pyegeria/egeria_tech_client.py @@ -5,6 +5,7 @@ The Tech Client is a facade that provides a unified interface to the most commonly used technical OMVS modules. """ +from pyegeria.core._no_password import NO_PASSWORD, carried_password, password_as_given, resolve_password from pyegeria.omvs.action_author import ActionAuthor from pyegeria.omvs.actor_manager import ActorManager from pyegeria.omvs.asset_catalog import AssetCatalog @@ -82,7 +83,7 @@ def __init__( self.view_server = view_server or settings.Environment.egeria_view_server self.platform_url = platform_url or settings.Environment.egeria_platform_url self.user_id = user_id or settings.User_Profile.user_name - self.user_pwd = user_pwd or settings.User_Profile.user_pwd + self.user_pwd, self.use_default_password = resolve_password(user_pwd) # NO_PASSWORD: opt out self.token = token # See BaseServerClient._token_minted_for: None means caller-supplied. self._token_minted_for = None @@ -157,18 +158,11 @@ def __init__( self.NO_ELEMENTS_FOUND = NO_ELEMENTS_FOUND def _get_subclient(self, attr_name: str): - """Lazy-load and cache sub-clients.""" + """Lazy-load and cache sub-clients, each built to act for this client (helper_client, C3).""" if attr_name not in self._instantiated_clients: - client_cls = self._subclient_map[attr_name] - self._instantiated_clients[attr_name] = client_cls( - self.view_server, - self.platform_url, - self.user_id, - self.user_pwd, - self.token, - timeout=self.timeout, - ) - self._instantiated_clients[attr_name]._token_minted_for = self._token_minted_for + from pyegeria.core._helper_client import helper_client + self._instantiated_clients[attr_name] = helper_client( + self._subclient_map[attr_name], self, timeout=self.timeout) return self._instantiated_clients[attr_name] def __getattr__(self, name): diff --git a/pyegeria/models/element_status.py b/pyegeria/models/element_status.py new file mode 100644 index 00000000..19a94300 --- /dev/null +++ b/pyegeria/models/element_status.py @@ -0,0 +1,125 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright Contributors to the ODPi Egeria project. +"""The element statuses Egeria accepts in a `limitResultsByStatus` filter, and the check that keeps others out. + +`limitResultsByStatus` filters on an element's *header* status, Egeria's `ElementStatus` enum. It is not one of +the status *properties* some types carry (contentStatus, deploymentStatus, ...). A value from one of those (DRAFT, +APPROVED, UNDER_DEVELOPMENT, ...) used to reach Egeria and come back as a 400 ("Cannot deserialize value of type +ElementStatus from String "DRAFT""), seen 2026-11-13 from a find whose caller meant "draft governance +definitions". pyegeria now refuses it before sending, and says which status property the value belongs to and +where that property lives. + +Everything here is offline. Egeria's valid metadata values for these properties are declared per property (the +typeName makes no difference) and equal the Java enums, so a server read would give the same words with a new +way to fail (STATUS-VALID-VALUES-LAZ-2026-11-13). + +This module imports nothing from pyegeria, so the request-body models can use it without an import cycle. +""" +from __future__ import annotations + +import warnings +from typing import Iterable, List, NamedTuple, Optional + +#: Egeria's ElementStatus values, from ElementStatus.java at odpi/egeria 3c8970b656 (2026-09-21), the enum +#: in the build laz runs (002a3ef1, 2026-11-13). Older Egerias (5.x) accepted more (DRAFT, PROPOSED, +#: APPROVED, ...): when Egeria's enum changes, change this set and name the new source. +ELEMENT_STATUSES = frozenset({"UNKNOWN", "ACTIVE", "INVALID", "DELETED", "OTHER"}) +ELEMENT_STATUS_SOURCE = "ElementStatus.java @ odpi/egeria 3c8970b656" + +#: Element statuses no element can hold: every one of the 676 types on laz (2026-11-13) has +#: validElementStatusList [ACTIVE, DELETED]. They pass the check (Egeria accepts them) but match nothing. +UNHELD_ELEMENT_STATUSES = frozenset({"UNKNOWN", "INVALID", "OTHER"}) + + +class StatusProperty(NamedTuple): + """A status property some types carry, the values its enum allows, and where it lives.""" + name: str # the property, e.g. contentStatus + enum: str # the Java enum its values come from + where: str # where the property lives, for the hint + on_elements: bool # declared on an entity or classification (so a find can filter on it); else relationships + values: frozenset + + +#: The status properties a misplaced `limitResultsByStatus` value most likely belongs to. Values: the Java enums +#: in open-metadata-framework .../frameworks/openmetadata/enums (odpi/egeria 8de9572c4d), equal to laz's +#: get_valid_metadata_values and to the enums at bae9bcb1 (Architect, 2026-11-13). Where: the types that declare +#: each property (subtypes inherit), from laz's type system. Element-carried properties first. +STATUS_PROPERTIES = ( + StatusProperty("contentStatus", "ContentStatus", "AuthoredReferenceable, DataAsset", True, frozenset({ + "DRAFT", "PREPARED", "PROPOSED", "APPROVED", "REJECTED", "ACTIVE", "DEPRECATED", "OBSOLETE", "OTHER"})), + StatusProperty("deploymentStatus", "DeploymentStatus", + "DigitalProduct, Infrastructure, SoftwareCapability, SolutionComponent, the Promise classification", + True, frozenset({ + "PROPOSED", "UNDER_DEVELOPMENT", "DEVELOPMENT_COMPLETE", "APPROVED_FOR_DEPLOYMENT", "REJECTED", "STANDBY", + "ACTIVE", "DISABLED", "FAILED", "OTHER"})), + StatusProperty("activityStatus", "ActivityStatus", "Process; ActionTarget and NotificationSubscriber relationships", + True, frozenset({ + "REQUESTED", "APPROVED", "WAITING", "ACTIVATING", "IN_PROGRESS", "PAUSED", "FOR_INFO", "COMPLETED", "INVALID", + "IGNORED", "FAILED", "CANCELLED", "ABANDONED", "OTHER"})), + StatusProperty("membershipStatus", "CollectionMemberStatus", "collection memberships", False, frozenset({ + "UNKNOWN", "DISCOVERED", "PROPOSED", "IMPORTED", "VALIDATED", "DEPRECATED", "OBSOLETE", "OTHER"})), + StatusProperty("termRelationshipStatus", "TermRelationshipStatus", "term relationships", False, frozenset({ + "DRAFT", "ACTIVE", "DEPRECATED", "OBSOLETE", "OTHER"})), + StatusProperty("termAssignmentStatus", "TermAssignmentStatus", "semantic assignments", False, frozenset({ + "DISCOVERED", "PROPOSED", "IMPORTED", "VALIDATED", "DEPRECATED", "OBSOLETE", "OTHER"})), +) + + +class InvalidElementStatus(ValueError): + """A `limitResultsByStatus` value that isn't one of Egeria's element statuses. A ValueError, so a pydantic + validator reports it as a field error (which pyegeria turns into PyegeriaInvalidParameterException).""" + + +class UnheldElementStatusWarning(UserWarning): + """A `limitResultsByStatus` value Egeria accepts but no element can hold (UNKNOWN, INVALID, OTHER).""" + + +def _phrase(prop: StatusProperty, first: bool) -> str: + if prop.on_elements: + return f"{'a ' if first else 'or '}{prop.name}{' value' if first else ''} ({prop.where})" + return f"{'a ' if first else 'or '}{prop.name}{' value' if first else ''} on {prop.where}" + + +def _why(value: str) -> str: + props = [p for p in STATUS_PROPERTIES if value in p.values] + if not props: + return f"{value} is not an element status, nor a value of any status property" + words = f"{value} is " + ", ".join(_phrase(p, i == 0) for i, p in enumerate(props)) + ", not an element status" + filterable = [p.name for p in props if p.on_elements] + return words + (f"; filter on {' or '.join(filterable)} instead" if filterable else "") + + +def check_element_statuses(values: Optional[Iterable[str]], *, field: str = "limit_results_by_status" + ) -> Optional[List[str]]: + """`values` upper-cased, or InvalidElementStatus naming each value that isn't an element status, the status + property it belongs to and where that lives. None and an empty list pass through unchanged (no status + filter). A lone string counts as one value. UNKNOWN, INVALID and OTHER pass, with an UnheldElementStatusWarning: + Egeria accepts them, but no element can hold them. + + Under ``-W error`` (or a ``filterwarnings = error`` test setting) that warning becomes an exception. Raised in + the request-body validator, it is not a ValueError, so pydantic doesn't wrap it: it propagates raw as + UnheldElementStatusWarning, not as a ValidationError / PyegeriaInvalidParameterException.""" + if values is None: + return None + if isinstance(values, str): + values = [values] + out: List[str] = [] + bad: List[str] = [] + for v in values: + s = str(v).strip().upper() + if s in ELEMENT_STATUSES: + out.append(s) + else: + bad.append(_why(s or repr(v))) + if bad: + raise InvalidElementStatus( + f"{field}: {'; '.join(bad)}. Element statuses are {', '.join(sorted(ELEMENT_STATUSES))} " + f"({ELEMENT_STATUS_SOURCE}).") + unheld = [s for s in out if s in UNHELD_ELEMENT_STATUSES] + if unheld: + warnings.warn( + f"{field}: {', '.join(unheld)} {'is an element status' if len(unheld) == 1 else 'are element statuses'}" + f" Egeria accepts, but no type lets an element hold {'it' if len(unheld) == 1 else 'them'} (every " + f"type's validElementStatusList is ACTIVE, DELETED), so {'it matches' if len(unheld) == 1 else 'they match'}" + f" nothing.", UnheldElementStatusWarning, stacklevel=2) + return out diff --git a/pyegeria/models/models.py b/pyegeria/models/models.py index 3c721d0d..64f151c5 100644 --- a/pyegeria/models/models.py +++ b/pyegeria/models/models.py @@ -10,7 +10,9 @@ from enum import Enum, StrEnum from typing import Literal, Annotated, Any, Optional, Dict -from pydantic import BaseModel, Field, ConfigDict, root_validator, model_validator, AliasChoices, model_serializer +from pydantic import BaseModel, Field, ConfigDict, root_validator, model_validator, AliasChoices, model_serializer, field_validator + +from pyegeria.models.element_status import check_element_statuses @@ -909,12 +911,18 @@ class ResultsRequestBody(GetRequestBody): anchor_guid: str | None = None anchor_domain_name: str | None = None anchor_scope_guid: str | None = None - limit_results_by_status: list[str] | None = None # header status + limit_results_by_status: list[str] | None = None # header status (Egeria's ElementStatus) sequencing_order: SequencingOrder | None = None sequencing_property: str | None = None start_from: int = 0 page_size: int = 0 + @field_validator("limit_results_by_status", mode="before") + @classmethod + def _element_statuses_only(cls, value): + """Refuse a value that isn't an element status (e.g. DRAFT, a content status) before Egeria 400s on it.""" + return check_element_statuses(value) + class LevelIdentifierQueryBody(ResultsRequestBody): class_: Annotated[Literal["LevelIdentifierQueryProperties"], Field(alias="class")] return_specific_level: bool = True diff --git a/pyegeria/omvs/action_author.py b/pyegeria/omvs/action_author.py index c6fc7a2c..a37cf94b 100644 --- a/pyegeria/omvs/action_author.py +++ b/pyegeria/omvs/action_author.py @@ -5,6 +5,7 @@ This module contains the Action Author View Service client. """ +from pyegeria.core._no_password import NO_PASSWORD, carried_password, password_as_given, resolve_password import asyncio from typing import Annotated, Literal, Optional @@ -84,7 +85,7 @@ def __init__( self.view_server = view_server self.platform_url = platform_url self.user_id = user_id - self.user_pwd = user_pwd + self.user_pwd = password_as_given(user_pwd) self.url_marker = "action-author" @dynamic_catch diff --git a/pyegeria/omvs/actor_manager.py b/pyegeria/omvs/actor_manager.py index ec13efef..c79a118f 100644 --- a/pyegeria/omvs/actor_manager.py +++ b/pyegeria/omvs/actor_manager.py @@ -6,6 +6,7 @@ """ +from pyegeria.core._no_password import NO_PASSWORD, carried_password, password_as_given, resolve_password import asyncio from typing import Optional, Any @@ -82,7 +83,7 @@ def __init__(self, view_server: str = None, platform_url: str = None, user_id: s self.view_server = view_server self.platform_url = platform_url self.user_id = user_id - self.user_pwd = user_pwd + self.user_pwd = password_as_given(user_pwd) # Handle Optional parameters for parent class ServerClient.__init__(self, view_server, platform_url, user_id, user_pwd, token, timeout=timeout) diff --git a/pyegeria/omvs/asset_catalog.py b/pyegeria/omvs/asset_catalog.py index 39ac6085..e031bc44 100644 --- a/pyegeria/omvs/asset_catalog.py +++ b/pyegeria/omvs/asset_catalog.py @@ -9,6 +9,7 @@ """ +from pyegeria.core._no_password import NO_PASSWORD, carried_password, password_as_given, resolve_password import asyncio from loguru import logger @@ -120,7 +121,7 @@ def __init__(self, view_server: str, platform_url: str, user_id: str, user_pwd: self.view_server = view_server self.platform_url = platform_url self.user_id = user_id - self.user_pwd = user_pwd + self.user_pwd = password_as_given(user_pwd) # # Output helpers diff --git a/pyegeria/omvs/automated_curation.py b/pyegeria/omvs/automated_curation.py index 9344cf5b..a0c7e111 100644 --- a/pyegeria/omvs/automated_curation.py +++ b/pyegeria/omvs/automated_curation.py @@ -6,6 +6,7 @@ """ +from pyegeria.core._no_password import NO_PASSWORD, carried_password, password_as_given, resolve_password import asyncio import warnings from datetime import datetime @@ -88,7 +89,7 @@ def __init__( self.view_server = view_server self.platform_url = platform_url self.user_id = user_id - self.user_pwd = user_pwd + self.user_pwd = password_as_given(user_pwd) ServerClient.__init__(self, view_server, platform_url, user_id, user_pwd, token=token, timeout=timeout) self.ref_curation_command_base = f"{self.platform_url}/servers/{self.view_server}/api/open-metadata/automated-curation" @@ -2541,10 +2542,9 @@ def _governance_engine_client(self): from pyegeria.omvs.governance_engine import GovernanceEngine client = getattr(self, "_gov_engine_client", None) if client is None: - client = GovernanceEngine(self.view_server, self.platform_url, self.user_id, self.user_pwd, - token=self.token, timeout=getattr(self, "timeout", None), - legacy_fallback=True) - client._token_minted_for = self._token_minted_for + from pyegeria.core._helper_client import helper_client + client = helper_client(GovernanceEngine, self, timeout=getattr(self, "timeout", None), + legacy_fallback=True) self._gov_engine_client = client return client diff --git a/pyegeria/omvs/classification_explorer.py b/pyegeria/omvs/classification_explorer.py index 328a0d3a..db1418b1 100644 --- a/pyegeria/omvs/classification_explorer.py +++ b/pyegeria/omvs/classification_explorer.py @@ -194,7 +194,7 @@ async def _async_get_classified_elements_by( "forLineage": false, "forDuplicateProcessing": false, "metadataElementTypeName": "GlossaryTerm", - "limitResultsByStatus": [ "ACTIVE", "DRAFT"], + "limitResultsByStatus": ["ACTIVE"], "sequencingProperty": "???", "sequencingOrder": "LAST_UPDATE_RECENT", "returnSpecificLevel": true, @@ -257,7 +257,7 @@ def get_classified_elements_by( "forLineage": false, "forDuplicateProcessing": false, "metadataElementTypeName": "GlossaryTerm", - "limitResultsByStatus": [ "ACTIVE", "DRAFT"], + "limitResultsByStatus": ["ACTIVE"], "sequencingProperty": "???", "sequencingOrder": "LAST_UPDATE_RECENT", "returnSpecificLevel": true, @@ -313,7 +313,7 @@ async def _async_get_security_tagged_elements( "forLineage" : false, "forDuplicateProcessing" : false, "metadataElementTypeName" : "GlossaryTerm", - "limitResultsByStatus" : [ "ACTIVE", "DRAFT"], + "limitResultsByStatus" : ["ACTIVE"], "sequencingProperty" : "???", "sequencingOrder" : "LAST_UPDATE_RECENT", "securityLabels" : [ "???" ], @@ -385,7 +385,7 @@ def get_security_tagged_elements( "forLineage" : false, "forDuplicateProcessing" : false, "metadataElementTypeName" : "GlossaryTerm", - "limitResultsByStatus" : [ "ACTIVE", "DRAFT"], + "limitResultsByStatus" : ["ACTIVE"], "sequencingProperty" : "???", "sequencingOrder" : "LAST_UPDATE_RECENT", "securityLabels" : [ "???" ], @@ -453,7 +453,7 @@ async def _async_get_owners_elements( "forLineage": false, "forDuplicateProcessing": false, "metadataElementTypeName": "GlossaryTerm", - "limitResultsByStatus": [ "ACTIVE", "DRAFT"], + "limitResultsByStatus": ["ACTIVE"], "sequencingProperty": "???", "sequencingOrder": "LAST_UPDATE_RECENT" } @@ -521,7 +521,7 @@ def get_owners_elements( "forLineage": false, "forDuplicateProcessing": false, "metadataElementTypeName": "GlossaryTerm", - "limitResultsByStatus": [ "ACTIVE", "DRAFT"], + "limitResultsByStatus": ["ACTIVE"], "sequencingProperty": "???", "sequencingOrder": "LAST_UPDATE_RECENT" } @@ -705,7 +705,7 @@ async def _async_get_subject_area_members( "forLineage": false, "forDuplicateProcessing": false, "metadataElementTypeName": "GlossaryTerm", - "limitResultsByStatus": [ "ACTIVE", "DRAFT"], + "limitResultsByStatus": ["ACTIVE"], "sequencingProperty": "???", "sequencingOrder": "LAST_UPDATE_RECENT" } @@ -774,7 +774,7 @@ def get_subject_area_members( "forLineage": false, "forDuplicateProcessing": false, "metadataElementTypeName": "GlossaryTerm", - "limitResultsByStatus": [ "ACTIVE", "DRAFT"], + "limitResultsByStatus": ["ACTIVE"], "sequencingProperty": "???", "sequencingOrder": "LAST_UPDATE_RECENT" } @@ -832,7 +832,7 @@ async def _async_get_elements_by_origin( "forLineage" : false, "forDuplicateProcessing" : false, "metadataElementTypeName" : "GlossaryTerm", - "limitResultsByStatus" : [ "ACTIVE", "DRAFT"], + "limitResultsByStatus" : ["ACTIVE"], "sequencingProperty" : "???", "sequencingOrder" : "LAST_UPDATE_RECENT", "organizationGUID" : "????", @@ -903,7 +903,7 @@ def get_elements_by_origin( "forLineage" : false, "forDuplicateProcessing" : false, "metadataElementTypeName" : "GlossaryTerm", - "limitResultsByStatus" : [ "ACTIVE", "DRAFT"], + "limitResultsByStatus" : ["ACTIVE"], "sequencingProperty" : "???", "sequencingOrder" : "LAST_UPDATE_RECENT", "organizationGUID" : "????", @@ -964,7 +964,7 @@ async def _async_get_meanings( "forLineage" : false, "forDuplicateProcessing" : false, "metadataElementTypeName" : "GlossaryTerm", - "limitResultsByStatus" : [ "ACTIVE", "DRAFT"], + "limitResultsByStatus" : ["ACTIVE"], "sequencingProperty" : "???", "sequencingOrder" : "LAST_UPDATE_RECENT", "expression" : "????", @@ -1040,7 +1040,7 @@ def get_meanings( "forLineage" : false, "forDuplicateProcessing" : false, "metadataElementTypeName" : "GlossaryTerm", - "limitResultsByStatus" : [ "ACTIVE", "DRAFT"], + "limitResultsByStatus" : ["ACTIVE"], "sequencingProperty" : "???", "sequencingOrder" : "LAST_UPDATE_RECENT", "expression" : "????", @@ -1105,7 +1105,7 @@ async def _async_get_semantic_assignees( "forLineage" : false, "forDuplicateProcessing" : false, "metadataElementTypeName" : "GlossaryTerm", - "limitResultsByStatus" : [ "ACTIVE", "DRAFT"], + "limitResultsByStatus" : ["ACTIVE"], "sequencingProperty" : "???", "sequencingOrder" : "LAST_UPDATE_RECENT", "expression" : "????", @@ -1181,7 +1181,7 @@ def get_semantic_assignees( "forLineage" : false, "forDuplicateProcessing" : false, "metadataElementTypeName" : "GlossaryTerm", - "limitResultsByStatus" : [ "ACTIVE", "DRAFT"], + "limitResultsByStatus" : ["ACTIVE"], "sequencingProperty" : "???", "sequencingOrder" : "LAST_UPDATE_RECENT", "expression" : "????", @@ -1256,7 +1256,7 @@ async def _async_get_governed_elements( "forLineage": false, "forDuplicateProcessing": false, "metadataElementTypeName": "GlossaryTerm", - "limitResultsByStatus": [ "ACTIVE", "DRAFT"], + "limitResultsByStatus": ["ACTIVE"], "sequencingProperty": "???", "sequencingOrder": "LAST_UPDATE_RECENT" } @@ -1324,7 +1324,7 @@ def get_governed_elements( "forLineage": false, "forDuplicateProcessing": false, "metadataElementTypeName": "GlossaryTerm", - "limitResultsByStatus": [ "ACTIVE", "DRAFT"], + "limitResultsByStatus": ["ACTIVE"], "sequencingProperty": "???", "sequencingOrder": "LAST_UPDATE_RECENT" } @@ -1390,7 +1390,7 @@ async def _async_get_governed_by_definitions( "forLineage": false, "forDuplicateProcessing": false, "metadataElementTypeName": "GlossaryTerm", - "limitResultsByStatus": [ "ACTIVE", "DRAFT"], + "limitResultsByStatus": ["ACTIVE"], "sequencingProperty": "???", "sequencingOrder": "LAST_UPDATE_RECENT" } @@ -1458,7 +1458,7 @@ def get_governed_by_definitions( "forLineage": false, "forDuplicateProcessing": false, "metadataElementTypeName": "GlossaryTerm", - "limitResultsByStatus": [ "ACTIVE", "DRAFT"], + "limitResultsByStatus": ["ACTIVE"], "sequencingProperty": "???", "sequencingOrder": "LAST_UPDATE_RECENT" } @@ -1527,7 +1527,7 @@ async def _async_get_source_elements( "forLineage": false, "forDuplicateProcessing": false, "metadataElementTypeName": "GlossaryTerm", - "limitResultsByStatus": [ "ACTIVE", "DRAFT"], + "limitResultsByStatus": ["ACTIVE"], "sequencingProperty": "???", "sequencingOrder": "LAST_UPDATE_RECENT" } @@ -1598,7 +1598,7 @@ def get_source_elements( "forLineage": false, "forDuplicateProcessing": false, "metadataElementTypeName": "GlossaryTerm", - "limitResultsByStatus": [ "ACTIVE", "DRAFT"], + "limitResultsByStatus": ["ACTIVE"], "sequencingProperty": "???", "sequencingOrder": "LAST_UPDATE_RECENT" } @@ -1666,7 +1666,7 @@ async def _async_get_elements_sourced_from( "forLineage": false, "forDuplicateProcessing": false, "metadataElementTypeName": "GlossaryTerm", - "limitResultsByStatus": [ "ACTIVE", "DRAFT"], + "limitResultsByStatus": ["ACTIVE"], "sequencingProperty": "???", "sequencingOrder": "LAST_UPDATE_RECENT" } @@ -1736,7 +1736,7 @@ def get_elements_sourced_from( "forLineage": false, "forDuplicateProcessing": false, "metadataElementTypeName": "GlossaryTerm", - "limitResultsByStatus": [ "ACTIVE", "DRAFT"], + "limitResultsByStatus": ["ACTIVE"], "sequencingProperty": "???", "sequencingOrder": "LAST_UPDATE_RECENT" } @@ -1802,7 +1802,7 @@ async def _async_get_scopes( "forLineage": false, "forDuplicateProcessing": false, "metadataElementTypeName": "GlossaryTerm", - "limitResultsByStatus": [ "ACTIVE", "DRAFT"], + "limitResultsByStatus": ["ACTIVE"], "sequencingProperty": "???", "sequencingOrder": "LAST_UPDATE_RECENT" } @@ -1870,7 +1870,7 @@ def get_scopes( "forLineage": false, "forDuplicateProcessing": false, "metadataElementTypeName": "GlossaryTerm", - "limitResultsByStatus": [ "ACTIVE", "DRAFT"], + "limitResultsByStatus": ["ACTIVE"], "sequencingProperty": "???", "sequencingOrder": "LAST_UPDATE_RECENT" } @@ -1936,7 +1936,7 @@ async def _async_get_scoped_elements( "forLineage": false, "forDuplicateProcessing": false, "metadataElementTypeName": "GlossaryTerm", - "limitResultsByStatus": [ "ACTIVE", "DRAFT"], + "limitResultsByStatus": ["ACTIVE"], "sequencingProperty": "???", "sequencingOrder": "LAST_UPDATE_RECENT" } @@ -2009,7 +2009,7 @@ def get_scoped_elements( "forLineage": false, "forDuplicateProcessing": false, "metadataElementTypeName": "GlossaryTerm", - "limitResultsByStatus": [ "ACTIVE", "DRAFT"], + "limitResultsByStatus": ["ACTIVE"], "sequencingProperty": "???", "sequencingOrder": "LAST_UPDATE_RECENT" } @@ -2073,7 +2073,7 @@ async def _async_get_licensed_elements( "forLineage": false, "forDuplicateProcessing": false, "metadataElementTypeName": "GlossaryTerm", - "limitResultsByStatus": [ "ACTIVE", "DRAFT"], + "limitResultsByStatus": ["ACTIVE"], "sequencingProperty": "???", "sequencingOrder": "LAST_UPDATE_RECENT" } @@ -2141,7 +2141,7 @@ def get_licensed_elements( "forLineage": false, "forDuplicateProcessing": false, "metadataElementTypeName": "GlossaryTerm", - "limitResultsByStatus": [ "ACTIVE", "DRAFT"], + "limitResultsByStatus": ["ACTIVE"], "sequencingProperty": "???", "sequencingOrder": "LAST_UPDATE_RECENT" } @@ -2207,7 +2207,7 @@ async def _async_get_licenses( "forLineage": false, "forDuplicateProcessing": false, "metadataElementTypeName": "GlossaryTerm", - "limitResultsByStatus": [ "ACTIVE", "DRAFT"], + "limitResultsByStatus": ["ACTIVE"], "sequencingProperty": "???", "sequencingOrder": "LAST_UPDATE_RECENT" } @@ -2275,7 +2275,7 @@ def get_licenses( "forLineage": false, "forDuplicateProcessing": false, "metadataElementTypeName": "GlossaryTerm", - "limitResultsByStatus": [ "ACTIVE", "DRAFT"], + "limitResultsByStatus": ["ACTIVE"], "sequencingProperty": "???", "sequencingOrder": "LAST_UPDATE_RECENT" } @@ -2341,7 +2341,7 @@ async def _async_get_certified_elements( "forLineage": false, "forDuplicateProcessing": false, "metadataElementTypeName": "GlossaryTerm", - "limitResultsByStatus": [ "ACTIVE", "DRAFT"], + "limitResultsByStatus": ["ACTIVE"], "sequencingProperty": "???", "sequencingOrder": "LAST_UPDATE_RECENT" } @@ -2409,7 +2409,7 @@ def get_certified_elements( "forLineage": false, "forDuplicateProcessing": false, "metadataElementTypeName": "GlossaryTerm", - "limitResultsByStatus": [ "ACTIVE", "DRAFT"], + "limitResultsByStatus": ["ACTIVE"], "sequencingProperty": "???", "sequencingOrder": "LAST_UPDATE_RECENT" } @@ -2476,7 +2476,7 @@ async def _async_get_certifications( "forLineage": false, "forDuplicateProcessing": false, "metadataElementTypeName": "GlossaryTerm", - "limitResultsByStatus": [ "ACTIVE", "DRAFT"], + "limitResultsByStatus": ["ACTIVE"], "sequencingProperty": "???", "sequencingOrder": "LAST_UPDATE_RECENT" } @@ -2544,7 +2544,7 @@ def get_certifications( "forLineage": false, "forDuplicateProcessing": false, "metadataElementTypeName": "GlossaryTerm", - "limitResultsByStatus": [ "ACTIVE", "DRAFT"], + "limitResultsByStatus": ["ACTIVE"], "sequencingProperty": "???", "sequencingOrder": "LAST_UPDATE_RECENT" } @@ -5050,7 +5050,7 @@ async def _async_get_relationships( response: Response = await self._async_make_request( "POST", url, body_slimmer(body), **kwargs ) - rels = response.json().get("relationships", "No relationships found") + rels = self._elements_noting_page_cap(response, "No relationships found", key="relationships") if type(rels) is list and len(rels) == 0: return NO_ELEMENTS_FOUND @@ -5198,7 +5198,7 @@ async def _async_get_relationships_with_property_value( response: Response = await self._async_make_request( "POST", url, body_slimmer(body), timeout=timeout, **kwargs ) - rels = response.json().get("relationships", NO_ELEMENTS_FOUND) + rels = self._elements_noting_page_cap(response, NO_ELEMENTS_FOUND, key="relationships") if type(rels) is list and len(rels) == 0: return NO_ELEMENTS_FOUND if output_format != 'JSON': # return a simplified markdown representation @@ -7265,7 +7265,7 @@ async def _async_get_resource_list( "forLineage": false, "forDuplicateProcessing": false, "metadataElementTypeName": "Referenceable", - "limitResultsByStatus": [ "ACTIVE", "DRAFT"], + "limitResultsByStatus": ["ACTIVE"], "sequencingProperty": "???", "sequencingOrder": "LAST_UPDATE_RECENT" } @@ -7334,7 +7334,7 @@ def get_resource_list( "forLineage": false, "forDuplicateProcessing": false, "metadataElementTypeName": "Referenceable", - "limitResultsByStatus": [ "ACTIVE", "DRAFT"], + "limitResultsByStatus": ["ACTIVE"], "sequencingProperty": "???", "sequencingOrder": "LAST_UPDATE_RECENT" } @@ -7402,7 +7402,7 @@ async def _async_get_supported_by_resource( "forLineage": false, "forDuplicateProcessing": false, "metadataElementTypeName": "Referenceable", - "limitResultsByStatus": [ "ACTIVE", "DRAFT"], + "limitResultsByStatus": ["ACTIVE"], "sequencingProperty": "???", "sequencingOrder": "LAST_UPDATE_RECENT" } @@ -7472,7 +7472,7 @@ def get_supported_by_resource( "forLineage": false, "forDuplicateProcessing": false, "metadataElementTypeName": "Referenceable", - "limitResultsByStatus": [ "ACTIVE", "DRAFT"], + "limitResultsByStatus": ["ACTIVE"], "sequencingProperty": "???", "sequencingOrder": "LAST_UPDATE_RECENT" } @@ -12496,9 +12496,8 @@ def _collection_client(self): from pyegeria.omvs.collection_manager import CollectionManager client = getattr(self, "_collection_mgr", None) if client is None: - client = CollectionManager(self.view_server, self.platform_url, self.user_id, self.user_pwd, - token=self.token, timeout=getattr(self, "timeout", None)) - client._token_minted_for = self._token_minted_for + from pyegeria.core._helper_client import helper_client + client = helper_client(CollectionManager, self, timeout=getattr(self, "timeout", None)) self._collection_mgr = client return client @@ -12827,7 +12826,7 @@ async def _async_find_root_elements( json_body = validated_body.model_dump_json(indent=2, exclude_none=True, by_alias=True) response = await self._async_make_request("POST", url, json_body, timeout=timeout) - elements = response.json().get("elements", NO_ELEMENTS_FOUND) + elements = self._elements_noting_page_cap(response, NO_ELEMENTS_FOUND) if type(elements) is str or len(elements) == 0: logger.info(NO_ELEMENTS_FOUND) diff --git a/pyegeria/omvs/collection_manager.py b/pyegeria/omvs/collection_manager.py index d5567c28..2e9c08b4 100644 --- a/pyegeria/omvs/collection_manager.py +++ b/pyegeria/omvs/collection_manager.py @@ -684,7 +684,7 @@ async def _async_find_digital_products( json_body = validated_body.model_dump_json(indent=2, exclude_none=True, by_alias=True) response = await self._async_make_request("POST", url, json_body) - elements = response.json().get("elements", NO_ELEMENTS_FOUND) + elements = self._elements_noting_page_cap(response, NO_ELEMENTS_FOUND) if type(elements) is str or len(elements) == 0: logger.info(NO_ELEMENTS_FOUND) @@ -811,7 +811,7 @@ async def _async_get_digital_products_by_category( json_body = validated_body.model_dump_json(indent=2, exclude_none=True, by_alias=True) response = await self._async_make_request("POST", url, json_body) - elements = response.json().get("elements", NO_ELEMENTS_FOUND) + elements = self._elements_noting_page_cap(response, NO_ELEMENTS_FOUND) if type(elements) is str or len(elements) == 0: logger.info(NO_ELEMENTS_FOUND) diff --git a/pyegeria/omvs/community_matters_omvs.py b/pyegeria/omvs/community_matters_omvs.py index e9c63696..61f16c56 100644 --- a/pyegeria/omvs/community_matters_omvs.py +++ b/pyegeria/omvs/community_matters_omvs.py @@ -11,6 +11,7 @@ validation and transport. """ +from pyegeria.core._no_password import NO_PASSWORD, carried_password, password_as_given, resolve_password import asyncio from typing import Any, Optional @@ -61,7 +62,7 @@ def __init__( self.view_server = view_server self.platform_url = platform_url self.user_id = user_id - self.user_pwd = user_pwd + self.user_pwd = password_as_given(user_pwd) self.community_command_base: str = ( f"{self.platform_url}/servers/{self.view_server}/api/open-metadata/community-matters" ) diff --git a/pyegeria/omvs/data_discovery.py b/pyegeria/omvs/data_discovery.py index 66b76b00..2384eb69 100644 --- a/pyegeria/omvs/data_discovery.py +++ b/pyegeria/omvs/data_discovery.py @@ -5,6 +5,7 @@ This module contains the Data Discovery View Service client. """ +from pyegeria.core._no_password import NO_PASSWORD, carried_password, password_as_given, resolve_password import asyncio import uuid from typing import Optional @@ -67,7 +68,7 @@ def __init__( self.view_server = view_server self.platform_url = platform_url self.user_id = user_id - self.user_pwd = user_pwd + self.user_pwd = password_as_given(user_pwd) self.url_marker = "data-discovery" def _extract_annotation_properties(self, element: dict, columns_struct: dict) -> dict: @@ -1974,9 +1975,8 @@ def _asset_maker_client(self): from pyegeria.omvs.asset_maker import AssetMaker client = getattr(self, "_asset_maker", None) if client is None: - client = AssetMaker(self.view_server, self.platform_url, self.user_id, self.user_pwd, - token=self.token, timeout=getattr(self, "timeout", None)) - client._token_minted_for = self._token_minted_for + from pyegeria.core._helper_client import helper_client + client = helper_client(AssetMaker, self, timeout=getattr(self, "timeout", None)) self._asset_maker = client return client diff --git a/pyegeria/omvs/data_engineer.py b/pyegeria/omvs/data_engineer.py index 70fccb93..92501150 100644 --- a/pyegeria/omvs/data_engineer.py +++ b/pyegeria/omvs/data_engineer.py @@ -5,6 +5,7 @@ Access tabular data sets """ +from pyegeria.core._no_password import NO_PASSWORD, carried_password, password_as_given, resolve_password import asyncio from pydantic import HttpUrl @@ -67,7 +68,7 @@ def __init__( self.view_server = view_server self.platform_url = platform_url self.user_id = user_id - self.user_pwd = user_pwd + self.user_pwd = password_as_given(user_pwd) ServerClient.__init__( self, view_server, platform_url, user_id, user_pwd, token, timeout=timeout diff --git a/pyegeria/omvs/lineage_linker.py b/pyegeria/omvs/lineage_linker.py index 02bfcf93..81639ecb 100644 --- a/pyegeria/omvs/lineage_linker.py +++ b/pyegeria/omvs/lineage_linker.py @@ -5,6 +5,7 @@ This module contains the Lineage Linker View Service client. """ +from pyegeria.core._no_password import NO_PASSWORD, carried_password, password_as_given, resolve_password import asyncio from typing import Annotated, Literal, Optional @@ -160,7 +161,7 @@ def __init__( self.view_server = view_server self.platform_url = platform_url self.user_id = user_id - self.user_pwd = user_pwd + self.user_pwd = password_as_given(user_pwd) self.url_marker = "lineage-linker" @dynamic_catch diff --git a/pyegeria/omvs/metadata_expert.py b/pyegeria/omvs/metadata_expert.py index 69598229..5e51697e 100644 --- a/pyegeria/omvs/metadata_expert.py +++ b/pyegeria/omvs/metadata_expert.py @@ -5,6 +5,7 @@ Merged Metadata Expert and Explorer OMVS client. """ +from pyegeria.core._no_password import NO_PASSWORD, carried_password, password_as_given, resolve_password import asyncio from loguru import logger from typing import Any, Optional, Union, Dict, List @@ -102,7 +103,7 @@ def __init__(self, view_server: str, platform_url: str, user_id: str, user_pwd: self.view_server = view_server self.platform_url = platform_url self.user_id = user_id - self.user_pwd = user_pwd + self.user_pwd = password_as_given(user_pwd) ServerClient.__init__(self, view_server, platform_url, user_id, user_pwd, token, timeout=timeout, **kwargs) logger.debug(f"{self.__class__.__name__} initialized") self.command_root: str = ( @@ -3757,7 +3758,7 @@ async def _async_find_metadata_elements( "POST", url, body_slimmer(body), timeout=timeout ) - elements = response.json().get("elements", NO_ELEMENTS_FOUND) + elements = self._elements_noting_page_cap(response, NO_ELEMENTS_FOUND) if type(elements) is list: if len(elements) == 0: return NO_ELEMENTS_FOUND diff --git a/pyegeria/omvs/people_organizer.py b/pyegeria/omvs/people_organizer.py index db972aa1..31fdcccd 100644 --- a/pyegeria/omvs/people_organizer.py +++ b/pyegeria/omvs/people_organizer.py @@ -5,6 +5,7 @@ This module contains the People Organizer View Service client. """ +from pyegeria.core._no_password import NO_PASSWORD, carried_password, password_as_given, resolve_password import asyncio from typing import Annotated, Literal, Optional @@ -55,7 +56,7 @@ def __init__( self.view_server = view_server self.platform_url = platform_url self.user_id = user_id - self.user_pwd = user_pwd + self.user_pwd = password_as_given(user_pwd) self.url_marker = "people-organizer" @dynamic_catch diff --git a/pyegeria/omvs/project_manager.py b/pyegeria/omvs/project_manager.py index 63c79779..47c2ebca 100644 --- a/pyegeria/omvs/project_manager.py +++ b/pyegeria/omvs/project_manager.py @@ -589,7 +589,7 @@ async def _async_get_projects_by_classification_properties( } response = await self._async_make_request("POST", url, body, **kwargs) - elements = response.json().get("elements", NO_ELEMENTS_FOUND) + elements = self._elements_noting_page_cap(response, NO_ELEMENTS_FOUND) if type(elements) is str or len(elements) == 0: logger.info(NO_ELEMENTS_FOUND) return NO_ELEMENTS_FOUND @@ -707,7 +707,7 @@ async def _async_get_project_team( else: response = await self._async_make_request("POST", url, **kwargs) - elements = response.json().get("elements", NO_ELEMENTS_FOUND) + elements = self._elements_noting_page_cap(response, NO_ELEMENTS_FOUND) if type(elements) is str or len(elements) == 0: logger.info(NO_ELEMENTS_FOUND) return NO_ELEMENTS_FOUND @@ -802,7 +802,7 @@ async def _async_get_project_hierarchy( else: response = await self._async_make_request("POST", url, **kwargs) - elements = response.json().get("elements", NO_ELEMENTS_FOUND) + elements = self._elements_noting_page_cap(response, NO_ELEMENTS_FOUND) if type(elements) is str or len(elements) == 0: logger.info(NO_ELEMENTS_FOUND) return NO_ELEMENTS_FOUND diff --git a/pyegeria/omvs/registered_info.py b/pyegeria/omvs/registered_info.py index ee12f69a..c5c00c2b 100644 --- a/pyegeria/omvs/registered_info.py +++ b/pyegeria/omvs/registered_info.py @@ -13,6 +13,7 @@ from typing import Optional +from pyegeria.core._no_password import NO_PASSWORD, carried_password, password_as_given, resolve_password from pyegeria.core._server_client import ServerClient from pyegeria.view.base_report_formats import select_report_spec, get_report_spec_match from pyegeria.view.output_formatter import ( @@ -65,7 +66,7 @@ def __init__( self.view_server = view_server self.platform_url = platform_url self.user_id = user_id - self.user_pwd = user_pwd + self.user_pwd = password_as_given(user_pwd) self.reg_command_root = ( f"{self.platform_url}/open-metadata/platform-services/" f"server-platform/registered-services" @@ -125,14 +126,15 @@ def list_registered_svcs( # Choose a report spec columns_struct = None - if isinstance(report_spec, str): - columns_struct = select_report_spec(report_spec, output_format) - elif isinstance(report_spec, dict): - columns_struct = get_report_spec_match(report_spec, output_format) - else: - columns_struct = select_report_spec("Registered-Services", output_format) - if columns_struct is None: - columns_struct = select_report_spec("Default", output_format) + with self._spec_identity(): # EGERIA-tier reads as this client (spec-cache T2) + if isinstance(report_spec, str): + columns_struct = select_report_spec(report_spec, output_format) + elif isinstance(report_spec, dict): + columns_struct = get_report_spec_match(report_spec, output_format) + else: + columns_struct = select_report_spec("Registered-Services", output_format) + if columns_struct is None: + columns_struct = select_report_spec("Default", output_format) return self._generate_registered_info_output( elements=elements, @@ -179,14 +181,15 @@ def list_severity_definitions( return elements or "No severities found" columns_struct = None - if isinstance(report_spec, str): - columns_struct = select_report_spec(report_spec, output_format) - elif isinstance(report_spec, dict): - columns_struct = get_report_spec_match(report_spec, output_format) - else: - columns_struct = select_report_spec("Severity-Definitions", output_format) - if columns_struct is None: - columns_struct = select_report_spec("Default", output_format) + with self._spec_identity(): # EGERIA-tier reads as this client (spec-cache T2) + if isinstance(report_spec, str): + columns_struct = select_report_spec(report_spec, output_format) + elif isinstance(report_spec, dict): + columns_struct = get_report_spec_match(report_spec, output_format) + else: + columns_struct = select_report_spec("Severity-Definitions", output_format) + if columns_struct is None: + columns_struct = select_report_spec("Default", output_format) return self._generate_registered_info_output( elements=elements, @@ -232,14 +235,15 @@ def list_asset_types( return elements or "no types found" columns_struct = None - if isinstance(report_spec, str): - columns_struct = select_report_spec(report_spec, output_format) - elif isinstance(report_spec, dict): - columns_struct = get_report_spec_match(report_spec, output_format) - else: - columns_struct = select_report_spec("Asset-Types", output_format) - if columns_struct is None: - columns_struct = select_report_spec("Default", output_format) + with self._spec_identity(): # EGERIA-tier reads as this client (spec-cache T2) + if isinstance(report_spec, str): + columns_struct = select_report_spec(report_spec, output_format) + elif isinstance(report_spec, dict): + columns_struct = get_report_spec_match(report_spec, output_format) + else: + columns_struct = select_report_spec("Asset-Types", output_format) + if columns_struct is None: + columns_struct = select_report_spec("Default", output_format) return self._generate_registered_info_output( elements=elements, diff --git a/pyegeria/omvs/schema_maker.py b/pyegeria/omvs/schema_maker.py index 8484db4e..4d77c6a0 100644 --- a/pyegeria/omvs/schema_maker.py +++ b/pyegeria/omvs/schema_maker.py @@ -5,6 +5,7 @@ This module contains the Schema Maker View Service client. """ +from pyegeria.core._no_password import NO_PASSWORD, carried_password, password_as_given, resolve_password import asyncio from typing import Annotated, Literal, Optional @@ -111,7 +112,7 @@ def __init__( self.view_server = view_server self.platform_url = platform_url self.user_id = user_id - self.user_pwd = user_pwd + self.user_pwd = password_as_given(user_pwd) self.url_marker = "schema-maker" def _extract_schema_properties(self, element: dict, columns_struct: dict) -> dict: diff --git a/pyegeria/omvs/solution_architect.py b/pyegeria/omvs/solution_architect.py index b892f9c1..e67207f8 100644 --- a/pyegeria/omvs/solution_architect.py +++ b/pyegeria/omvs/solution_architect.py @@ -6,6 +6,7 @@ """ +from pyegeria.core._no_password import NO_PASSWORD, carried_password, password_as_given, resolve_password import asyncio import os import sys @@ -91,7 +92,7 @@ def __init__(self, view_server: str, platform_url: str, user_id: Optional[str] = self.view_server = view_server self.platform_url = platform_url self.user_id = user_id - self.user_pwd = user_pwd + self.user_pwd = password_as_given(user_pwd) self.solution_architect_command_root: str = (f"{self.platform_url}/servers/{self.view_server}" f"/api/open-metadata/solution-architect") ServerClient.__init__(self, view_server, platform_url, user_id=user_id, user_pwd=user_pwd, token=token, timeout=timeout) @@ -124,7 +125,8 @@ def _extract_design_pattern_properties(self, element: dict, columns_struct: dict def generate_design_pattern_output(self, elements: list | dict, search_string: Optional[str] = None, element_type_name: Optional[str] = None, output_format: str = 'MD', report_spec: dict | str | None = None, **kwargs): - report_spec = select_report_spec(report_spec, 'Design-Pattern-DrE') + with self._spec_identity(): # EGERIA-tier reads as this client (spec-cache T2) + report_spec = select_report_spec(report_spec, 'Design-Pattern-DrE') return self.generate_output(elements, self._extract_design_pattern_properties, report_spec, search_string=search_string, element_type_name=element_type_name, output_format=output_format, **kwargs) diff --git a/pyegeria/omvs/subject_area.py b/pyegeria/omvs/subject_area.py index 5db4e804..a2892125 100644 --- a/pyegeria/omvs/subject_area.py +++ b/pyegeria/omvs/subject_area.py @@ -5,6 +5,7 @@ This module contains the Subject Area View Service client. """ +from pyegeria.core._no_password import NO_PASSWORD, carried_password, password_as_given, resolve_password import asyncio from typing import Annotated, Literal, Optional @@ -74,7 +75,7 @@ def __init__( self.view_server = view_server self.platform_url = platform_url self.user_id = user_id - self.user_pwd = user_pwd + self.user_pwd = password_as_given(user_pwd) self.url_marker = "subject-area" def _extract_subject_area_properties(self, element: dict, columns_struct: dict) -> dict: diff --git a/pyegeria/omvs/time_keeper.py b/pyegeria/omvs/time_keeper.py index cdd41303..5b5b13ec 100644 --- a/pyegeria/omvs/time_keeper.py +++ b/pyegeria/omvs/time_keeper.py @@ -5,6 +5,7 @@ This module contains the Time Keeper View Service client. """ +from pyegeria.core._no_password import NO_PASSWORD, carried_password, password_as_given, resolve_password import asyncio from typing import Annotated, Literal, Optional @@ -118,7 +119,7 @@ def __init__( self.view_server = view_server self.platform_url = platform_url self.user_id = user_id - self.user_pwd = user_pwd + self.user_pwd = password_as_given(user_pwd) self.url_marker = "time-keeper" def _extract_context_event_properties(self, element: dict, columns_struct: dict) -> dict: diff --git a/pyegeria/omvs/valid_metadata.py b/pyegeria/omvs/valid_metadata.py index a9d4165a..b9706a3c 100644 --- a/pyegeria/omvs/valid_metadata.py +++ b/pyegeria/omvs/valid_metadata.py @@ -6,6 +6,7 @@ """ +from pyegeria.core._no_password import NO_PASSWORD, carried_password, password_as_given, resolve_password import asyncio import re from typing import Any, Optional @@ -58,7 +59,7 @@ def __init__( self.view_server = view_server self.platform_url = platform_url self.user_id = user_id - self.user_pwd = user_pwd + self.user_pwd = password_as_given(user_pwd) self.valid_m_command_base: str = f"/api/open-metadata/valid-metadata" self.page_size = max_paging_size diff --git a/pyegeria/view/base_report_formats.py b/pyegeria/view/base_report_formats.py index 5149259d..b0f3c73c 100644 --- a/pyegeria/view/base_report_formats.py +++ b/pyegeria/view/base_report_formats.py @@ -79,7 +79,8 @@ import os import re as _re -from datetime import datetime, timezone +import warnings +from datetime import datetime, timedelta, timezone from importlib.metadata import entry_points from pathlib import Path from typing import Any, Callable, Dict, List, Optional, Tuple, Union @@ -316,7 +317,7 @@ 'Regulation-DrE-Basic': FormatSet(target_type='Regulation', heading='Regulation-DrE-Basic Attributes', description='Auto-generated format for Regulation (Create, Basic).', family='Governance Officer', formats=[Format(types=['LIST'], attributes=[Column(name='Display Name', key='display_name'), Column(name='Qualified Name', key='qualified_name'), Column(name='GUID', key='guid'), Column(name='Zone Membership', key='zone_membership'), Column(name='Description', key='description'), Column(name='Legal', key='legal'), Column(name='Authors', key='authors'), Column(name='Domain Identifier', key='domain_identifier', valid_values=['All Domains', 'Data', 'Privacy', 'Security', 'IT Infrastructure', 'Software Development', 'Corporate', 'Asset Management', 'Other']), Column(name='Implications', key='implications'), Column(name='Importance', key='importance'), Column(name='Outcomes', key='outcomes'), Column(name='Results', key='results'), Column(name='Scope', key='scope'), Column(name='Summary', key='summary'), Column(name='Usage', key='usage'), Column(name='Regulation Source', key='regulation_source'), Column(name='Regulators', key='regulators'), Column(name='Version Identifier', key='version_identifier'), Column(name='Content Status', key='content_status', valid_values=['DRAFT', 'PREPARED', 'PROPOSED', 'APPROVED', 'REJECTED', 'ACTIVE', 'DEPRECATED', 'OTHER']), Column(name='Category', key='category')]), Format(types=['ALL'], attributes=[Column(name='Zone Membership', key='zone_membership'), Column(name='Category', key='category'), Column(name='Description', key='description'), Column(name='Display Name', key='display_name'), Column(name='GUID', key='guid'), Column(name='Legal', key='legal'), Column(name='Qualified Name', key='qualified_name'), Column(name='URL', key='url'), Column(name='Version Identifier', key='version_identifier'), Column(name='Authors', key='authors'), Column(name='Content Status', key='content_status', valid_values=['DRAFT', 'PREPARED', 'PROPOSED', 'APPROVED', 'REJECTED', 'ACTIVE', 'DEPRECATED', 'OTHER']), Column(name='Domain Identifier', key='domain_identifier', valid_values=['All Domains', 'Data', 'Privacy', 'Security', 'IT Infrastructure', 'Software Development', 'Corporate', 'Asset Management', 'Other']), Column(name='Implications', key='implications'), Column(name='Importance', key='importance'), Column(name='Outcomes', key='outcomes'), Column(name='Results', key='results'), Column(name='Scope', key='scope'), Column(name='Summary', key='summary'), Column(name='Usage', key='usage'), Column(name='Regulation Source', key='regulation_source'), Column(name='Regulators', key='regulators')])], action=ActionParameter(function='GovernanceOfficer.find_governance_definitions', required_params=['search_string'], optional_params=['sequencing_order', 'sequencing_property', 'page_size', 'start_from', 'starts_with', 'ends_with', 'ignore_case', 'classification_names', 'metadata_element_subtypes', 'metadata_element_type'], spec_params={'metadata_element_type': 'Regulation'})), 'Related-Media-DrE-Advanced': FormatSet(target_type='Related Media', heading='Related-Media-DrE-Advanced Attributes', description='Auto-generated format for Related Media (Create, Advanced).', family='External Reference', formats=[Format(types=['LIST'], attributes=[Column(name='Display Name', key='display_name'), Column(name='Qualified Name', key='qualified_name'), Column(name='GUID', key='guid'), Column(name='Effective From', key='effective_from'), Column(name='Effective Time', key='effective_time'), Column(name='Effective To', key='effective_to'), Column(name='Glossary Term', key='glossary_term'), Column(name='Is Own Anchor', key='is_own_anchor'), Column(name='Zone Membership', key='zone_membership'), Column(name='Additional Properties', key='additional_properties'), Column(name='Confidence Classification', key='confidence_classification', valid_values=['UNCLASSIFIED', 'AD_HOC', 'TRANSACTIONAL', 'AUTHORITATIVE', 'DERIVED', 'OBSOLETE', 'OTHER']), Column(name='Confidentiality Classification', key='confidentiality_classification', valid_values=['UNCLASSIFIED', 'INTERNAL', 'CONFIDENTIAL', 'SENSITIVE', 'RESTRICTED', 'OTHER']), Column(name='Criticality Classification', key='criticality_classification', valid_values=['UNCLASSIFIED', 'MARGINAL', 'IMPORTANT', 'CRITICAL', 'CATASTROPHIC', 'OTHER']), Column(name='Description', key='description'), Column(name='Identifier', key='identifier'), Column(name='Impact Classification', key='impact_classification', valid_values=['UNCLASSIFIED', 'LOW', 'MEDIUM', 'HIGH', 'OTHER']), Column(name='Legal', key='legal'), Column(name='Policy Management Point', key='policy_management_point'), Column(name='Retention Classification', key='retention_classification', valid_values=['UNCLASSIFIED', 'TEMPORARY', 'PROJECT_LIFETIME', 'TEAM_LIFETIME', 'CONTRACT_LIFETIME', 'REGULATED_LIFETIME', 'TIMEBOXED_LIFETIME', 'OTHER']), Column(name='Security Tags', key='security_tags'), Column(name='Authors', key='authors'), Column(name='Attribution', key='attribution'), Column(name='Copyright', key='copyright'), Column(name='License', key='license'), Column(name='Organization', key='organization'), Column(name='Reference Abstract', key='reference_abstract'), Column(name='Reference Title', key='reference_title'), Column(name='Sources', key='reference_sources'), Column(name='Default Media Usage', key='default_media_usage', valid_values=['ICON', 'THUMBNAIL', 'ILLUSTRATION', 'USAGE_GUIDANCE', 'OTHER']), Column(name='Default Media Usage Other Id', key='default_media_usage_other_id'), Column(name='Media Type', key='media_type', valid_values=['IMAGE', 'AUDIO', 'DOCUMENT', 'VIDEO', 'OTHER']), Column(name='Media Type Other Id', key='media_type_other_id'), Column(name='Classifications', key='classifications'), Column(name='Anchor Scope Name', key='anchor_scope_guid'), Column(name='Supplementary Properties', key='supplementary_properties'), Column(name='Version Identifier', key='version_identifier'), Column(name='Status', key='status'), Column(name='Content Status', key='content_status', valid_values=['DRAFT', 'PREPARED', 'PROPOSED', 'APPROVED', 'REJECTED', 'ACTIVE', 'DEPRECATED', 'OTHER']), Column(name='User Defined Content Status', key='user_defined_content_status'), Column(name='User Defined Status', key='user_defined_status'), Column(name='Category', key='category')]), Format(types=['ALL'], attributes=[Column(name='Effective From', key='effective_from'), Column(name='Effective Time', key='effective_time'), Column(name='Effective To', key='effective_to'), Column(name='Glossary Term', key='glossary_term'), Column(name='Is Own Anchor', key='is_own_anchor'), Column(name='Status', key='status'), Column(name='Zone Membership', key='zone_membership'), Column(name='Additional Properties', key='additional_properties'), Column(name='Category', key='category'), Column(name='Confidence Classification', key='confidence_classification', valid_values=['UNCLASSIFIED', 'AD_HOC', 'TRANSACTIONAL', 'AUTHORITATIVE', 'DERIVED', 'OBSOLETE', 'OTHER']), Column(name='Confidentiality Classification', key='confidentiality_classification', valid_values=['UNCLASSIFIED', 'INTERNAL', 'CONFIDENTIAL', 'SENSITIVE', 'RESTRICTED', 'OTHER']), Column(name='Criticality Classification', key='criticality_classification', valid_values=['UNCLASSIFIED', 'MARGINAL', 'IMPORTANT', 'CRITICAL', 'CATASTROPHIC', 'OTHER']), Column(name='Description', key='description'), Column(name='Display Name', key='display_name'), Column(name='GUID', key='guid'), Column(name='Identifier', key='identifier'), Column(name='Impact Classification', key='impact_classification', valid_values=['UNCLASSIFIED', 'LOW', 'MEDIUM', 'HIGH', 'OTHER']), Column(name='Legal', key='legal'), Column(name='Policy Management Point', key='policy_management_point'), Column(name='Qualified Name', key='qualified_name'), Column(name='Retention Classification', key='retention_classification', valid_values=['UNCLASSIFIED', 'TEMPORARY', 'PROJECT_LIFETIME', 'TEAM_LIFETIME', 'CONTRACT_LIFETIME', 'REGULATED_LIFETIME', 'TIMEBOXED_LIFETIME', 'OTHER']), Column(name='Security Tags', key='security_tags'), Column(name='URL', key='url'), Column(name='Version Identifier', key='version_identifier'), Column(name='Authors', key='authors'), Column(name='Content Status', key='content_status', valid_values=['DRAFT', 'PREPARED', 'PROPOSED', 'APPROVED', 'REJECTED', 'ACTIVE', 'DEPRECATED', 'OTHER']), Column(name='User Defined Content Status', key='user_defined_content_status'), Column(name='Attribution', key='attribution'), Column(name='Copyright', key='copyright'), Column(name='License', key='license'), Column(name='Organization', key='organization'), Column(name='Reference Abstract', key='reference_abstract'), Column(name='Reference Title', key='reference_title'), Column(name='Sources', key='reference_sources'), Column(name='Default Media Usage', key='default_media_usage', valid_values=['ICON', 'THUMBNAIL', 'ILLUSTRATION', 'USAGE_GUIDANCE', 'OTHER']), Column(name='Default Media Usage Other Id', key='default_media_usage_other_id'), Column(name='Media Type', key='media_type', valid_values=['IMAGE', 'AUDIO', 'DOCUMENT', 'VIDEO', 'OTHER']), Column(name='Media Type Other Id', key='media_type_other_id'), Column(name='User Defined Status', key='user_defined_status'), Column(name='Classifications', key='classifications'), Column(name='Anchor Scope Name', key='anchor_scope_guid'), Column(name='Supplementary Properties', key='supplementary_properties')])], action=ActionParameter(function='ExternalReferences.find_external_references', required_params=['search_string'], optional_params=['sequencing_order', 'sequencing_property', 'page_size', 'start_from', 'starts_with', 'ends_with', 'ignore_case', 'classification_names', 'metadata_element_subtypes', 'metadata_element_type'], spec_params={'metadata_element_type': 'RelatedMedia'})), 'Related-Media-DrE-Basic': FormatSet(target_type='Related Media', heading='Related-Media-DrE-Basic Attributes', description='Auto-generated format for Related Media (Create, Basic).', family='External Reference', formats=[Format(types=['LIST'], attributes=[Column(name='Display Name', key='display_name'), Column(name='Qualified Name', key='qualified_name'), Column(name='GUID', key='guid'), Column(name='Zone Membership', key='zone_membership'), Column(name='Description', key='description'), Column(name='Legal', key='legal'), Column(name='Authors', key='authors'), Column(name='Attribution', key='attribution'), Column(name='Copyright', key='copyright'), Column(name='License', key='license'), Column(name='Organization', key='organization'), Column(name='Reference Abstract', key='reference_abstract'), Column(name='Reference Title', key='reference_title'), Column(name='Sources', key='reference_sources'), Column(name='Default Media Usage', key='default_media_usage', valid_values=['ICON', 'THUMBNAIL', 'ILLUSTRATION', 'USAGE_GUIDANCE', 'OTHER']), Column(name='Default Media Usage Other Id', key='default_media_usage_other_id'), Column(name='Media Type', key='media_type', valid_values=['IMAGE', 'AUDIO', 'DOCUMENT', 'VIDEO', 'OTHER']), Column(name='Media Type Other Id', key='media_type_other_id'), Column(name='Version Identifier', key='version_identifier'), Column(name='Content Status', key='content_status', valid_values=['DRAFT', 'PREPARED', 'PROPOSED', 'APPROVED', 'REJECTED', 'ACTIVE', 'DEPRECATED', 'OTHER']), Column(name='Category', key='category')]), Format(types=['ALL'], attributes=[Column(name='Zone Membership', key='zone_membership'), Column(name='Category', key='category'), Column(name='Description', key='description'), Column(name='Display Name', key='display_name'), Column(name='GUID', key='guid'), Column(name='Legal', key='legal'), Column(name='Qualified Name', key='qualified_name'), Column(name='URL', key='url'), Column(name='Version Identifier', key='version_identifier'), Column(name='Authors', key='authors'), Column(name='Content Status', key='content_status', valid_values=['DRAFT', 'PREPARED', 'PROPOSED', 'APPROVED', 'REJECTED', 'ACTIVE', 'DEPRECATED', 'OTHER']), Column(name='Attribution', key='attribution'), Column(name='Copyright', key='copyright'), Column(name='License', key='license'), Column(name='Organization', key='organization'), Column(name='Reference Abstract', key='reference_abstract'), Column(name='Reference Title', key='reference_title'), Column(name='Sources', key='reference_sources'), Column(name='Default Media Usage', key='default_media_usage', valid_values=['ICON', 'THUMBNAIL', 'ILLUSTRATION', 'USAGE_GUIDANCE', 'OTHER']), Column(name='Default Media Usage Other Id', key='default_media_usage_other_id'), Column(name='Media Type', key='media_type', valid_values=['IMAGE', 'AUDIO', 'DOCUMENT', 'VIDEO', 'OTHER']), Column(name='Media Type Other Id', key='media_type_other_id')])], action=ActionParameter(function='ExternalReferences.find_external_references', required_params=['search_string'], optional_params=['sequencing_order', 'sequencing_property', 'page_size', 'start_from', 'starts_with', 'ends_with', 'ignore_case', 'classification_names', 'metadata_element_subtypes', 'metadata_element_type'], spec_params={'metadata_element_type': 'RelatedMedia'})), - 'Report-DrE-Advanced': FormatSet(target_type='Report', heading='Report-DrE-Advanced Attributes', description='Auto-generated format for Report (Create, Advanced).', family='Views & Reports', formats=[Format(types=['LIST'], attributes=[Column(name='Display Name', key='display_name'), Column(name='Qualified Name', key='qualified_name'), Column(name='GUID', key='guid'), Column(name='Effective From', key='effective_from'), Column(name='Effective Time', key='effective_time'), Column(name='Effective To', key='effective_to'), Column(name='Glossary Term', key='glossary_term'), Column(name='Is Own Anchor', key='is_own_anchor'), Column(name='Zone Membership', key='zone_membership'), Column(name='Additional Properties', key='additional_properties'), Column(name='Confidence Classification', key='confidence_classification', valid_values=['UNCLASSIFIED', 'AD_HOC', 'TRANSACTIONAL', 'AUTHORITATIVE', 'DERIVED', 'OBSOLETE', 'OTHER']), Column(name='Confidentiality Classification', key='confidentiality_classification', valid_values=['UNCLASSIFIED', 'INTERNAL', 'CONFIDENTIAL', 'SENSITIVE', 'RESTRICTED', 'OTHER']), Column(name='Criticality Classification', key='criticality_classification', valid_values=['UNCLASSIFIED', 'MARGINAL', 'IMPORTANT', 'CRITICAL', 'CATASTROPHIC', 'OTHER']), Column(name='Description', key='description'), Column(name='Identifier', key='identifier'), Column(name='Impact Classification', key='impact_classification', valid_values=['UNCLASSIFIED', 'LOW', 'MEDIUM', 'HIGH', 'OTHER']), Column(name='Legal', key='legal'), Column(name='Policy Management Point', key='policy_management_point'), Column(name='Retention Classification', key='retention_classification', valid_values=['UNCLASSIFIED', 'TEMPORARY', 'PROJECT_LIFETIME', 'TEAM_LIFETIME', 'CONTRACT_LIFETIME', 'REGULATED_LIFETIME', 'TIMEBOXED_LIFETIME', 'OTHER']), Column(name='Security Tags', key='security_tags'), Column(name='Deployed Implementation Type', key='deployed_impl_type'), Column(name='Namespace Path', key='namespace_path'), Column(name='Resource Name', key='resource_name'), Column(name='Source', key='source'), Column(name='Analytic Parameters', key='analytic_parameters'), Column(name='AsOfTime', key='as_of_time'), Column(name='Ends With', key='ends_with'), Column(name='Governance Zone Filter', key='governance_zone_filter'), Column(name='Graph Query Depth', key='graph_query_depth'), Column(name='Ignore Case', key='ignore_case'), Column(name='Include Only Classified Elements', key='include_only_classified_elements'), Column(name='Include Only Relationships', key='include_only_relationships'), Column(name='Metadata Element Subtype Names', key='metadata_element_subtype_names'), Column(name='Metadata Element Type Name', key='metadata_element_type_name'), Column(name='Order Property Name', key='order_property_name'), Column(name='Output Format', key='output_format', valid_values=['BAR', 'DICT', 'FORM', 'GRAPH', 'HTML', 'JSON', 'KPI', 'LIST', 'MD', 'MERMAID', 'PIE', 'REPORT', 'REPORT-GRAPH', 'SERIES', 'TABLE']), Column(name='Output Sort Order', key='sort_order', valid_values=['ANY', 'CREATION_DATE_RECENT', 'CREATION_DATA_OLDEST', 'LAST_UPDATE_RECENT', 'LAST_UPDATE_OLDEST', 'PROPERTY_ASCENDING', 'PROPERTY_DESCENDING']), Column(name='Page Size', key='page_size'), Column(name='Report Parameters', key='report_parameters'), Column(name='Report Spec', key='report_spec'), Column(name='Search String', key='search_string'), Column(name='Skip Classified Elements', key='skip_classified_elements'), Column(name='Skip Relationships', key='skip_relationships'), Column(name='Start From', key='start_from'), Column(name='Starts With', key='starts_with'), Column(name='Classifications', key='classifications'), Column(name='Anchor Scope Name', key='anchor_scope_guid'), Column(name='Supplementary Properties', key='supplementary_properties'), Column(name='Version Identifier', key='version_identifier'), Column(name='Status', key='status'), Column(name='Limit Result by Status', key='limit_result_by_status', valid_values=['UNKNOWN', 'DRAFT', 'PREPARED', 'PROPOSED', 'APPROVED', 'REJECTED', 'APPROVED_CONCEPT', 'UNDER_DEVELOPMENT', 'DEVELOPMENT_COMPLETE', 'APPROVED_FOR_DEPLOYMENT', 'STANDBY', 'ACTIVE', 'FAILED', 'DISABLED', 'COMPLETE', 'DEPRECATED', 'OTHER', 'DELETED']), Column(name='User Defined Status', key='user_defined_status'), Column(name='Category', key='category')]), Format(types=['ALL'], attributes=[Column(name='Effective From', key='effective_from'), Column(name='Effective Time', key='effective_time'), Column(name='Effective To', key='effective_to'), Column(name='Glossary Term', key='glossary_term'), Column(name='Is Own Anchor', key='is_own_anchor'), Column(name='Status', key='status'), Column(name='Zone Membership', key='zone_membership'), Column(name='Additional Properties', key='additional_properties'), Column(name='Category', key='category'), Column(name='Confidence Classification', key='confidence_classification', valid_values=['UNCLASSIFIED', 'AD_HOC', 'TRANSACTIONAL', 'AUTHORITATIVE', 'DERIVED', 'OBSOLETE', 'OTHER']), Column(name='Confidentiality Classification', key='confidentiality_classification', valid_values=['UNCLASSIFIED', 'INTERNAL', 'CONFIDENTIAL', 'SENSITIVE', 'RESTRICTED', 'OTHER']), Column(name='Criticality Classification', key='criticality_classification', valid_values=['UNCLASSIFIED', 'MARGINAL', 'IMPORTANT', 'CRITICAL', 'CATASTROPHIC', 'OTHER']), Column(name='Description', key='description'), Column(name='Display Name', key='display_name'), Column(name='GUID', key='guid'), Column(name='Identifier', key='identifier'), Column(name='Impact Classification', key='impact_classification', valid_values=['UNCLASSIFIED', 'LOW', 'MEDIUM', 'HIGH', 'OTHER']), Column(name='Legal', key='legal'), Column(name='Policy Management Point', key='policy_management_point'), Column(name='Qualified Name', key='qualified_name'), Column(name='Retention Classification', key='retention_classification', valid_values=['UNCLASSIFIED', 'TEMPORARY', 'PROJECT_LIFETIME', 'TEAM_LIFETIME', 'CONTRACT_LIFETIME', 'REGULATED_LIFETIME', 'TIMEBOXED_LIFETIME', 'OTHER']), Column(name='Security Tags', key='security_tags'), Column(name='URL', key='url'), Column(name='Version Identifier', key='version_identifier'), Column(name='Deployed Implementation Type', key='deployed_impl_type'), Column(name='Namespace Path', key='namespace_path'), Column(name='Resource Name', key='resource_name'), Column(name='Source', key='source'), Column(name='Analytic Parameters', key='analytic_parameters'), Column(name='AsOfTime', key='as_of_time'), Column(name='Ends With', key='ends_with'), Column(name='Governance Zone Filter', key='governance_zone_filter'), Column(name='Graph Query Depth', key='graph_query_depth'), Column(name='Ignore Case', key='ignore_case'), Column(name='Include Only Classified Elements', key='include_only_classified_elements'), Column(name='Include Only Relationships', key='include_only_relationships'), Column(name='Limit Result by Status', key='limit_result_by_status', valid_values=['UNKNOWN', 'DRAFT', 'PREPARED', 'PROPOSED', 'APPROVED', 'REJECTED', 'APPROVED_CONCEPT', 'UNDER_DEVELOPMENT', 'DEVELOPMENT_COMPLETE', 'APPROVED_FOR_DEPLOYMENT', 'STANDBY', 'ACTIVE', 'FAILED', 'DISABLED', 'COMPLETE', 'DEPRECATED', 'OTHER', 'DELETED']), Column(name='Metadata Element Subtype Names', key='metadata_element_subtype_names'), Column(name='Metadata Element Type Name', key='metadata_element_type_name'), Column(name='Order Property Name', key='order_property_name'), Column(name='Output Format', key='output_format', valid_values=['BAR', 'DICT', 'FORM', 'GRAPH', 'HTML', 'JSON', 'KPI', 'LIST', 'MD', 'MERMAID', 'PIE', 'REPORT', 'REPORT-GRAPH', 'SERIES', 'TABLE']), Column(name='Output Sort Order', key='sort_order', valid_values=['ANY', 'CREATION_DATE_RECENT', 'CREATION_DATA_OLDEST', 'LAST_UPDATE_RECENT', 'LAST_UPDATE_OLDEST', 'PROPERTY_ASCENDING', 'PROPERTY_DESCENDING']), Column(name='Page Size', key='page_size'), Column(name='Report Parameters', key='report_parameters'), Column(name='Report Spec', key='report_spec'), Column(name='Search String', key='search_string'), Column(name='Skip Classified Elements', key='skip_classified_elements'), Column(name='Skip Relationships', key='skip_relationships'), Column(name='Start From', key='start_from'), Column(name='Starts With', key='starts_with'), Column(name='User Defined Status', key='user_defined_status'), Column(name='Classifications', key='classifications'), Column(name='Anchor Scope Name', key='anchor_scope_guid'), Column(name='Supplementary Properties', key='supplementary_properties')])], action=ActionParameter(function='ClassificationExplorer.find_elements_by_property_value', required_params=['search_string'], optional_params=['sequencing_order', 'sequencing_property', 'page_size', 'start_from', 'starts_with', 'ends_with', 'ignore_case', 'classification_names', 'metadata_element_subtypes', 'metadata_element_type'], spec_params={'metadata_element_type': 'Report', 'property_names': ['displayName', 'qualifiedName']})), + 'Report-DrE-Advanced': FormatSet(target_type='Report', heading='Report-DrE-Advanced Attributes', description='Auto-generated format for Report (Create, Advanced).', family='Views & Reports', formats=[Format(types=['LIST'], attributes=[Column(name='Display Name', key='display_name'), Column(name='Qualified Name', key='qualified_name'), Column(name='GUID', key='guid'), Column(name='Effective From', key='effective_from'), Column(name='Effective Time', key='effective_time'), Column(name='Effective To', key='effective_to'), Column(name='Glossary Term', key='glossary_term'), Column(name='Is Own Anchor', key='is_own_anchor'), Column(name='Zone Membership', key='zone_membership'), Column(name='Additional Properties', key='additional_properties'), Column(name='Confidence Classification', key='confidence_classification', valid_values=['UNCLASSIFIED', 'AD_HOC', 'TRANSACTIONAL', 'AUTHORITATIVE', 'DERIVED', 'OBSOLETE', 'OTHER']), Column(name='Confidentiality Classification', key='confidentiality_classification', valid_values=['UNCLASSIFIED', 'INTERNAL', 'CONFIDENTIAL', 'SENSITIVE', 'RESTRICTED', 'OTHER']), Column(name='Criticality Classification', key='criticality_classification', valid_values=['UNCLASSIFIED', 'MARGINAL', 'IMPORTANT', 'CRITICAL', 'CATASTROPHIC', 'OTHER']), Column(name='Description', key='description'), Column(name='Identifier', key='identifier'), Column(name='Impact Classification', key='impact_classification', valid_values=['UNCLASSIFIED', 'LOW', 'MEDIUM', 'HIGH', 'OTHER']), Column(name='Legal', key='legal'), Column(name='Policy Management Point', key='policy_management_point'), Column(name='Retention Classification', key='retention_classification', valid_values=['UNCLASSIFIED', 'TEMPORARY', 'PROJECT_LIFETIME', 'TEAM_LIFETIME', 'CONTRACT_LIFETIME', 'REGULATED_LIFETIME', 'TIMEBOXED_LIFETIME', 'OTHER']), Column(name='Security Tags', key='security_tags'), Column(name='Deployed Implementation Type', key='deployed_impl_type'), Column(name='Namespace Path', key='namespace_path'), Column(name='Resource Name', key='resource_name'), Column(name='Source', key='source'), Column(name='Analytic Parameters', key='analytic_parameters'), Column(name='AsOfTime', key='as_of_time'), Column(name='Ends With', key='ends_with'), Column(name='Governance Zone Filter', key='governance_zone_filter'), Column(name='Graph Query Depth', key='graph_query_depth'), Column(name='Ignore Case', key='ignore_case'), Column(name='Include Only Classified Elements', key='include_only_classified_elements'), Column(name='Include Only Relationships', key='include_only_relationships'), Column(name='Metadata Element Subtype Names', key='metadata_element_subtype_names'), Column(name='Metadata Element Type Name', key='metadata_element_type_name'), Column(name='Order Property Name', key='order_property_name'), Column(name='Output Format', key='output_format', valid_values=['BAR', 'DICT', 'FORM', 'GRAPH', 'HTML', 'JSON', 'KPI', 'LIST', 'MD', 'MERMAID', 'PIE', 'REPORT', 'REPORT-GRAPH', 'SERIES', 'TABLE']), Column(name='Output Sort Order', key='sort_order', valid_values=['ANY', 'CREATION_DATE_RECENT', 'CREATION_DATA_OLDEST', 'LAST_UPDATE_RECENT', 'LAST_UPDATE_OLDEST', 'PROPERTY_ASCENDING', 'PROPERTY_DESCENDING']), Column(name='Page Size', key='page_size'), Column(name='Report Parameters', key='report_parameters'), Column(name='Report Spec', key='report_spec'), Column(name='Search String', key='search_string'), Column(name='Skip Classified Elements', key='skip_classified_elements'), Column(name='Skip Relationships', key='skip_relationships'), Column(name='Start From', key='start_from'), Column(name='Starts With', key='starts_with'), Column(name='Classifications', key='classifications'), Column(name='Anchor Scope Name', key='anchor_scope_guid'), Column(name='Supplementary Properties', key='supplementary_properties'), Column(name='Version Identifier', key='version_identifier'), Column(name='Status', key='status'), Column(name='Limit Result by Status', key='limit_result_by_status', valid_values=['UNKNOWN', 'ACTIVE', 'INVALID', 'DELETED', 'OTHER']), Column(name='User Defined Status', key='user_defined_status'), Column(name='Category', key='category')]), Format(types=['ALL'], attributes=[Column(name='Effective From', key='effective_from'), Column(name='Effective Time', key='effective_time'), Column(name='Effective To', key='effective_to'), Column(name='Glossary Term', key='glossary_term'), Column(name='Is Own Anchor', key='is_own_anchor'), Column(name='Status', key='status'), Column(name='Zone Membership', key='zone_membership'), Column(name='Additional Properties', key='additional_properties'), Column(name='Category', key='category'), Column(name='Confidence Classification', key='confidence_classification', valid_values=['UNCLASSIFIED', 'AD_HOC', 'TRANSACTIONAL', 'AUTHORITATIVE', 'DERIVED', 'OBSOLETE', 'OTHER']), Column(name='Confidentiality Classification', key='confidentiality_classification', valid_values=['UNCLASSIFIED', 'INTERNAL', 'CONFIDENTIAL', 'SENSITIVE', 'RESTRICTED', 'OTHER']), Column(name='Criticality Classification', key='criticality_classification', valid_values=['UNCLASSIFIED', 'MARGINAL', 'IMPORTANT', 'CRITICAL', 'CATASTROPHIC', 'OTHER']), Column(name='Description', key='description'), Column(name='Display Name', key='display_name'), Column(name='GUID', key='guid'), Column(name='Identifier', key='identifier'), Column(name='Impact Classification', key='impact_classification', valid_values=['UNCLASSIFIED', 'LOW', 'MEDIUM', 'HIGH', 'OTHER']), Column(name='Legal', key='legal'), Column(name='Policy Management Point', key='policy_management_point'), Column(name='Qualified Name', key='qualified_name'), Column(name='Retention Classification', key='retention_classification', valid_values=['UNCLASSIFIED', 'TEMPORARY', 'PROJECT_LIFETIME', 'TEAM_LIFETIME', 'CONTRACT_LIFETIME', 'REGULATED_LIFETIME', 'TIMEBOXED_LIFETIME', 'OTHER']), Column(name='Security Tags', key='security_tags'), Column(name='URL', key='url'), Column(name='Version Identifier', key='version_identifier'), Column(name='Deployed Implementation Type', key='deployed_impl_type'), Column(name='Namespace Path', key='namespace_path'), Column(name='Resource Name', key='resource_name'), Column(name='Source', key='source'), Column(name='Analytic Parameters', key='analytic_parameters'), Column(name='AsOfTime', key='as_of_time'), Column(name='Ends With', key='ends_with'), Column(name='Governance Zone Filter', key='governance_zone_filter'), Column(name='Graph Query Depth', key='graph_query_depth'), Column(name='Ignore Case', key='ignore_case'), Column(name='Include Only Classified Elements', key='include_only_classified_elements'), Column(name='Include Only Relationships', key='include_only_relationships'), Column(name='Limit Result by Status', key='limit_result_by_status', valid_values=['UNKNOWN', 'ACTIVE', 'INVALID', 'DELETED', 'OTHER']), Column(name='Metadata Element Subtype Names', key='metadata_element_subtype_names'), Column(name='Metadata Element Type Name', key='metadata_element_type_name'), Column(name='Order Property Name', key='order_property_name'), Column(name='Output Format', key='output_format', valid_values=['BAR', 'DICT', 'FORM', 'GRAPH', 'HTML', 'JSON', 'KPI', 'LIST', 'MD', 'MERMAID', 'PIE', 'REPORT', 'REPORT-GRAPH', 'SERIES', 'TABLE']), Column(name='Output Sort Order', key='sort_order', valid_values=['ANY', 'CREATION_DATE_RECENT', 'CREATION_DATA_OLDEST', 'LAST_UPDATE_RECENT', 'LAST_UPDATE_OLDEST', 'PROPERTY_ASCENDING', 'PROPERTY_DESCENDING']), Column(name='Page Size', key='page_size'), Column(name='Report Parameters', key='report_parameters'), Column(name='Report Spec', key='report_spec'), Column(name='Search String', key='search_string'), Column(name='Skip Classified Elements', key='skip_classified_elements'), Column(name='Skip Relationships', key='skip_relationships'), Column(name='Start From', key='start_from'), Column(name='Starts With', key='starts_with'), Column(name='User Defined Status', key='user_defined_status'), Column(name='Classifications', key='classifications'), Column(name='Anchor Scope Name', key='anchor_scope_guid'), Column(name='Supplementary Properties', key='supplementary_properties')])], action=ActionParameter(function='ClassificationExplorer.find_elements_by_property_value', required_params=['search_string'], optional_params=['sequencing_order', 'sequencing_property', 'page_size', 'start_from', 'starts_with', 'ends_with', 'ignore_case', 'classification_names', 'metadata_element_subtypes', 'metadata_element_type'], spec_params={'metadata_element_type': 'Report', 'property_names': ['displayName', 'qualifiedName']})), 'Report-DrE-Basic': FormatSet(target_type='Report', heading='Report-DrE-Basic Attributes', description='Auto-generated format for Report (Create, Basic).', family='Views & Reports', formats=[Format(types=['LIST'], attributes=[Column(name='Display Name', key='display_name'), Column(name='Qualified Name', key='qualified_name'), Column(name='GUID', key='guid'), Column(name='Zone Membership', key='zone_membership'), Column(name='Description', key='description'), Column(name='Legal', key='legal'), Column(name='Deployed Implementation Type', key='deployed_impl_type'), Column(name='Namespace Path', key='namespace_path'), Column(name='Resource Name', key='resource_name'), Column(name='Analytic Parameters', key='analytic_parameters'), Column(name='Ends With', key='ends_with'), Column(name='Ignore Case', key='ignore_case'), Column(name='Metadata Element Subtype Names', key='metadata_element_subtype_names'), Column(name='Metadata Element Type Name', key='metadata_element_type_name'), Column(name='Output Format', key='output_format', valid_values=['BAR', 'DICT', 'FORM', 'GRAPH', 'HTML', 'JSON', 'KPI', 'LIST', 'MD', 'MERMAID', 'PIE', 'REPORT', 'REPORT-GRAPH', 'SERIES', 'TABLE']), Column(name='Page Size', key='page_size'), Column(name='Report Parameters', key='report_parameters'), Column(name='Report Spec', key='report_spec'), Column(name='Search String', key='search_string'), Column(name='Start From', key='start_from'), Column(name='Starts With', key='starts_with'), Column(name='Version Identifier', key='version_identifier'), Column(name='Category', key='category')]), Format(types=['ALL'], attributes=[Column(name='Zone Membership', key='zone_membership'), Column(name='Category', key='category'), Column(name='Description', key='description'), Column(name='Display Name', key='display_name'), Column(name='GUID', key='guid'), Column(name='Legal', key='legal'), Column(name='Qualified Name', key='qualified_name'), Column(name='URL', key='url'), Column(name='Version Identifier', key='version_identifier'), Column(name='Deployed Implementation Type', key='deployed_impl_type'), Column(name='Namespace Path', key='namespace_path'), Column(name='Resource Name', key='resource_name'), Column(name='Analytic Parameters', key='analytic_parameters'), Column(name='Ends With', key='ends_with'), Column(name='Ignore Case', key='ignore_case'), Column(name='Metadata Element Subtype Names', key='metadata_element_subtype_names'), Column(name='Metadata Element Type Name', key='metadata_element_type_name'), Column(name='Output Format', key='output_format', valid_values=['BAR', 'DICT', 'FORM', 'GRAPH', 'HTML', 'JSON', 'KPI', 'LIST', 'MD', 'MERMAID', 'PIE', 'REPORT', 'REPORT-GRAPH', 'SERIES', 'TABLE']), Column(name='Page Size', key='page_size'), Column(name='Report Parameters', key='report_parameters'), Column(name='Report Spec', key='report_spec'), Column(name='Search String', key='search_string'), Column(name='Start From', key='start_from'), Column(name='Starts With', key='starts_with')])], action=ActionParameter(function='ClassificationExplorer.find_elements_by_property_value', required_params=['search_string'], optional_params=['sequencing_order', 'sequencing_property', 'page_size', 'start_from', 'starts_with', 'ends_with', 'ignore_case', 'classification_names', 'metadata_element_subtypes', 'metadata_element_type'], spec_params={'metadata_element_type': 'Report', 'property_names': ['displayName', 'qualifiedName']})), 'Report-Spec-DrE-Advanced': FormatSet(target_type='Report Spec', heading='Report-Spec-DrE-Advanced Attributes', description='Auto-generated format for Report Spec (Create, Advanced).', family='Views & Reports', formats=[Format(types=['LIST'], attributes=[Column(name='Display Name', key='display_name'), Column(name='Qualified Name', key='qualified_name'), Column(name='GUID', key='guid'), Column(name='Effective From', key='effective_from'), Column(name='Effective Time', key='effective_time'), Column(name='Effective To', key='effective_to'), Column(name='Glossary Term', key='glossary_term'), Column(name='Is Own Anchor', key='is_own_anchor'), Column(name='Zone Membership', key='zone_membership'), Column(name='Additional Properties', key='additional_properties'), Column(name='Confidence Classification', key='confidence_classification', valid_values=['UNCLASSIFIED', 'AD_HOC', 'TRANSACTIONAL', 'AUTHORITATIVE', 'DERIVED', 'OBSOLETE', 'OTHER']), Column(name='Confidentiality Classification', key='confidentiality_classification', valid_values=['UNCLASSIFIED', 'INTERNAL', 'CONFIDENTIAL', 'SENSITIVE', 'RESTRICTED', 'OTHER']), Column(name='Criticality Classification', key='criticality_classification', valid_values=['UNCLASSIFIED', 'MARGINAL', 'IMPORTANT', 'CRITICAL', 'CATASTROPHIC', 'OTHER']), Column(name='Description', key='description'), Column(name='Identifier', key='identifier'), Column(name='Impact Classification', key='impact_classification', valid_values=['UNCLASSIFIED', 'LOW', 'MEDIUM', 'HIGH', 'OTHER']), Column(name='Legal', key='legal'), Column(name='Policy Management Point', key='policy_management_point'), Column(name='Retention Classification', key='retention_classification', valid_values=['UNCLASSIFIED', 'TEMPORARY', 'PROJECT_LIFETIME', 'TEAM_LIFETIME', 'CONTRACT_LIFETIME', 'REGULATED_LIFETIME', 'TIMEBOXED_LIFETIME', 'OTHER']), Column(name='Security Tags', key='security_tags'), Column(name='Authors', key='authors'), Column(name='Purpose', key='purpose'), Column(name='Microflow', key='microflow'), Column(name='Target Type Name', key='target_type_name'), Column(name='Aliases', key='aliases'), Column(name='Output Types', key='output_types'), Column(name='Include Preamble', key='include_preamble'), Column(name='Classifications', key='classifications'), Column(name='Anchor Scope Name', key='anchor_scope_guid'), Column(name='Supplementary Properties', key='supplementary_properties'), Column(name='Version Identifier', key='version_identifier'), Column(name='Status', key='status'), Column(name='Content Status', key='content_status', valid_values=['DRAFT', 'PREPARED', 'PROPOSED', 'APPROVED', 'REJECTED', 'ACTIVE', 'DEPRECATED', 'OTHER']), Column(name='User Defined Content Status', key='user_defined_content_status'), Column(name='User Defined Status', key='user_defined_status'), Column(name='Category', key='category')]), Format(types=['ALL'], attributes=[Column(name='Effective From', key='effective_from'), Column(name='Effective Time', key='effective_time'), Column(name='Effective To', key='effective_to'), Column(name='Glossary Term', key='glossary_term'), Column(name='Is Own Anchor', key='is_own_anchor'), Column(name='Status', key='status'), Column(name='Zone Membership', key='zone_membership'), Column(name='Additional Properties', key='additional_properties'), Column(name='Category', key='category'), Column(name='Confidence Classification', key='confidence_classification', valid_values=['UNCLASSIFIED', 'AD_HOC', 'TRANSACTIONAL', 'AUTHORITATIVE', 'DERIVED', 'OBSOLETE', 'OTHER']), Column(name='Confidentiality Classification', key='confidentiality_classification', valid_values=['UNCLASSIFIED', 'INTERNAL', 'CONFIDENTIAL', 'SENSITIVE', 'RESTRICTED', 'OTHER']), Column(name='Criticality Classification', key='criticality_classification', valid_values=['UNCLASSIFIED', 'MARGINAL', 'IMPORTANT', 'CRITICAL', 'CATASTROPHIC', 'OTHER']), Column(name='Description', key='description'), Column(name='Display Name', key='display_name'), Column(name='GUID', key='guid'), Column(name='Identifier', key='identifier'), Column(name='Impact Classification', key='impact_classification', valid_values=['UNCLASSIFIED', 'LOW', 'MEDIUM', 'HIGH', 'OTHER']), Column(name='Legal', key='legal'), Column(name='Policy Management Point', key='policy_management_point'), Column(name='Qualified Name', key='qualified_name'), Column(name='Retention Classification', key='retention_classification', valid_values=['UNCLASSIFIED', 'TEMPORARY', 'PROJECT_LIFETIME', 'TEAM_LIFETIME', 'CONTRACT_LIFETIME', 'REGULATED_LIFETIME', 'TIMEBOXED_LIFETIME', 'OTHER']), Column(name='Security Tags', key='security_tags'), Column(name='URL', key='url'), Column(name='Version Identifier', key='version_identifier'), Column(name='Authors', key='authors'), Column(name='Content Status', key='content_status', valid_values=['DRAFT', 'PREPARED', 'PROPOSED', 'APPROVED', 'REJECTED', 'ACTIVE', 'DEPRECATED', 'OTHER']), Column(name='User Defined Content Status', key='user_defined_content_status'), Column(name='Purpose', key='purpose'), Column(name='Microflow', key='microflow'), Column(name='Target Type Name', key='target_type_name'), Column(name='Aliases', key='aliases'), Column(name='Output Types', key='output_types'), Column(name='Include Preamble', key='include_preamble'), Column(name='User Defined Status', key='user_defined_status'), Column(name='Classifications', key='classifications'), Column(name='Anchor Scope Name', key='anchor_scope_guid'), Column(name='Supplementary Properties', key='supplementary_properties')])], action=ActionParameter(function='CollectionManager.find_collections', required_params=['search_string'], optional_params=['sequencing_order', 'sequencing_property', 'page_size', 'start_from', 'starts_with', 'ends_with', 'ignore_case', 'classification_names', 'metadata_element_subtypes', 'metadata_element_type'], spec_params={'metadata_element_type': 'ReportSpec'})), 'Report-Spec-DrE-Basic': FormatSet(target_type='Report Spec', heading='Report-Spec-DrE-Basic Attributes', description='Auto-generated format for Report Spec (Create, Basic).', family='Views & Reports', formats=[Format(types=['LIST'], attributes=[Column(name='Display Name', key='display_name'), Column(name='Qualified Name', key='qualified_name'), Column(name='GUID', key='guid'), Column(name='Zone Membership', key='zone_membership'), Column(name='Description', key='description'), Column(name='Legal', key='legal'), Column(name='Authors', key='authors'), Column(name='Purpose', key='purpose'), Column(name='Microflow', key='microflow'), Column(name='Target Type Name', key='target_type_name'), Column(name='Aliases', key='aliases'), Column(name='Output Types', key='output_types'), Column(name='Include Preamble', key='include_preamble'), Column(name='Version Identifier', key='version_identifier'), Column(name='Content Status', key='content_status', valid_values=['DRAFT', 'PREPARED', 'PROPOSED', 'APPROVED', 'REJECTED', 'ACTIVE', 'DEPRECATED', 'OTHER']), Column(name='Category', key='category')]), Format(types=['ALL'], attributes=[Column(name='Zone Membership', key='zone_membership'), Column(name='Category', key='category'), Column(name='Description', key='description'), Column(name='Display Name', key='display_name'), Column(name='GUID', key='guid'), Column(name='Legal', key='legal'), Column(name='Qualified Name', key='qualified_name'), Column(name='URL', key='url'), Column(name='Version Identifier', key='version_identifier'), Column(name='Authors', key='authors'), Column(name='Content Status', key='content_status', valid_values=['DRAFT', 'PREPARED', 'PROPOSED', 'APPROVED', 'REJECTED', 'ACTIVE', 'DEPRECATED', 'OTHER']), Column(name='Purpose', key='purpose'), Column(name='Microflow', key='microflow'), Column(name='Target Type Name', key='target_type_name'), Column(name='Aliases', key='aliases'), Column(name='Output Types', key='output_types'), Column(name='Include Preamble', key='include_preamble')])], action=ActionParameter(function='CollectionManager.find_collections', required_params=['search_string'], optional_params=['sequencing_order', 'sequencing_property', 'page_size', 'start_from', 'starts_with', 'ends_with', 'ignore_case', 'classification_names', 'metadata_element_subtypes', 'metadata_element_type'], spec_params={'metadata_element_type': 'ReportSpec'})), @@ -3964,6 +3965,41 @@ def report_spec_markdown() -> str: # Default TTL is 1 day; override with PYEGERIA_REPORT_SPECS_CACHE_TTL (seconds). _EGERIA_SPECS_LOADED_AT: Optional[datetime] = None _EGERIA_SPECS_CACHE_TTL: int = int(os.getenv("PYEGERIA_REPORT_SPECS_CACHE_TTL", str(86400))) +#: After a read Egeria didn't confirm complete (pager step 6), the next load retries after this long; what was +#: read stays registered. Only a confirmed end sets _EGERIA_SPECS_LOADED_AT. +_EGERIA_SPECS_RETRY_SECONDS: int = 300 +_EGERIA_SPECS_RETRY_AT: Optional[datetime] = None +#: The current retry delay after an unconfirmed read: 300 s, doubling each time, capped at the cache TTL, reset by a +#: confirmed load (the legacy-loader ruling, 2026-11-14). +_EGERIA_SPECS_RETRY_DELAY: Optional[int] = None +_LEGACY_LOADER_WARNED = False +LEGACY_LOADER_DEPRECATION = ("load_egeria_report_specs is deprecated and will be removed in pyegeria 7 (expected " + "early January 2027): it is the legacy ReportType:: loader. Use " + "pyegeria.view.egeria_spec_cache.sync_report_specs instead.") +#: How the last load_egeria_report_specs call ended (85f5bc15 read-back R1), beside its bool return: +#: {"status": one of REPORT_SPECS_LOAD_STATUSES, "count": ReportTypes read or None, "retry_at": ISO or None, +#: "loaded_at": ISO or None, "error": words or None}. +REPORT_SPECS_LOAD_STATUSES = ("loaded", "registered_unconfirmed", "fresh", "retry_window", "failed") +_EGERIA_SPECS_LAST_STATUS: Dict[str, Any] = {"status": None} + + +def report_specs_load_status() -> Dict[str, Any]: + """How the last ``load_egeria_report_specs`` call ended (a copy), so a caller can word it: + - loaded: Egeria confirmed the end; the registry holds every ReportType it read (count may be 0: none); + - registered_unconfirmed: what was read is registered, but Egeria didn't confirm there are no more; the + next load re-reads after retry_at; + - fresh: the cache from a confirmed load is still within its TTL (no read); + - retry_window: the last read wasn't confirmed complete and retry_at hasn't passed (no read; force reads); + - failed: Egeria couldn't be queried (error says why); nothing was marked loaded.""" + return dict(_EGERIA_SPECS_LAST_STATUS) + + +def _set_load_status(status: str, *, count: Optional[int] = None, error: Optional[str] = None) -> None: + _EGERIA_SPECS_LAST_STATUS.clear() + _EGERIA_SPECS_LAST_STATUS.update( + status=status, count=count, error=error, + retry_at=_EGERIA_SPECS_RETRY_AT.isoformat() if _EGERIA_SPECS_RETRY_AT else None, + loaded_at=_EGERIA_SPECS_LOADED_AT.isoformat() if _EGERIA_SPECS_LOADED_AT else None) class ReportFormatCollision(ValueError): @@ -4763,7 +4799,15 @@ def load_egeria_report_specs( force: bool = False, ttl_seconds: Optional[int] = None, ) -> bool: - """Pull ReportTypes and their QuestionSpec folders from Egeria and merge + """DEPRECATED, and will be removed in pyegeria 7 (expected early January 2027). Use + ``pyegeria.view.egeria_spec_cache.sync_report_specs`` instead: the spec cache is the system of record for + Egeria-held report specs. It reads both the ``ReportSpec::`` and ``ReportType::`` markers and serves the + consumer index and the authoring API. This legacy loader (the May-2026 ``ReportType::`` path) reads + ``ReportType::`` collections only, so Egeria-held ``ReportSpec::`` specs never load through it. It is kept, + unchanged in what it searches, for one more release, so callers have time to move. A DeprecationWarning is + raised once per process. + + Pull ReportTypes and their QuestionSpec folders from Egeria and merge question_specs into the corresponding file-based FormatSets in the runtime registry. Results are cached for `ttl_seconds` (default: ``PYEGERIA_REPORT_SPECS_CACHE_TTL`` @@ -4796,8 +4840,15 @@ def load_egeria_report_specs( the file-based registry, a warning is logged and the Egeria version is ignored (file wins). """ - global _RUNTIME_REPORT_FORMATS, _EGERIA_SPECS_LOADED_AT - + global _RUNTIME_REPORT_FORMATS, _EGERIA_SPECS_LOADED_AT, _EGERIA_SPECS_RETRY_AT + + global _LEGACY_LOADER_WARNED, _EGERIA_SPECS_RETRY_DELAY + if not _LEGACY_LOADER_WARNED: + _LEGACY_LOADER_WARNED = True + # Python hides DeprecationWarning outside __main__ and test runners, so the same words are also logged + # once at WARNING (mcp_server and the Portal would never show the warning alone). + logger.warning(LEGACY_LOADER_DEPRECATION) + warnings.warn(LEGACY_LOADER_DEPRECATION, DeprecationWarning, stacklevel=2) effective_ttl = ttl_seconds if ttl_seconds is not None else _EGERIA_SPECS_CACHE_TTL if not force and _EGERIA_SPECS_LOADED_AT is not None: @@ -4806,47 +4857,60 @@ def load_egeria_report_specs( logger.debug( f"load_egeria_report_specs: cache is fresh ({age:.0f}s old, TTL={effective_ttl}s) — skipping reload" ) + _set_load_status("fresh") return False - # Paginate through all collections with "ReportType" in their qualified name. - # Use page_size=1000 (the platform maximum) to fetch all in as few calls as possible. - # Do NOT stop on `len(page) < _page_size` -- per Egeria's own paging contract - # (https://egeria-project.org/guides/developer/finding-metadata/overview/#paging), - # a short-but-nonempty page does not mean "last page": server-side filtering can - # legitimately shorten an individual page without ending the result set, and the - # documented termination signal is a genuinely empty/null result, not a short one. - # Confirmed live (2026-08-18, ISSUE-54 investigation) that a `Referenceable`-scoped - # find_metadata_elements page can come back well short of pageSize while thousands - # of further elements are still reachable at a later startFrom. Always advance by - # _page_size regardless of how many came back; the `if not page` check above is the - # only correct stop condition. - _page_size = 1000 - _start = 0 - report_types: list = [] + if not force and _EGERIA_SPECS_RETRY_AT is not None and datetime.now(timezone.utc) < _EGERIA_SPECS_RETRY_AT: + logger.debug("load_egeria_report_specs: the last read wasn't confirmed complete; retrying after " + f"{_EGERIA_SPECS_RETRY_AT.isoformat()}") + _set_load_status("retry_window") + return False + + # Every collection with "ReportType" in its qualified name, through pyegeria's pager (pager step 6): an + # empty page is a filtered one, not the end; startFrom advances by the page size; the read's end decides + # whether the load counts as done. find_collections is collection-manager by-search-string -> + # OpenMetadataHandlerBase.findRootElements -> findMetadataElementsWithString (not the findMetadataElements + # the step 4 capture covered), so it pages as a plain suspect path. + from pyegeria.core.paging import log_end, page_all, paging_for + _page_size = 1000 # the platform maximum: as few requests as possible try: - while True: - page = client.find_collections( - search_string="ReportType", - _type="Collection", - start_from=_start, - page_size=_page_size, - ) - if not page or isinstance(page, str): - break - # Keep only entries whose qualified name contains "ReportType::" - report_types.extend( - r for r in page - if "ReportType::" in r.get("properties", {}).get("qualifiedName", "") - ) - _start += _page_size + paged = page_all(lambda start, size: client.find_collections(search_string="ReportType", _type="Collection", + start_from=start, page_size=size), + fetch_page_size=_page_size, **paging_for("find_collections", client)) except Exception as e: - logger.warning(f"load_egeria_report_specs: could not query Egeria — skipping: {e}") + logger.warning(f"load_egeria_report_specs: could not query Egeria -- not loaded, will retry: {e}") + _set_load_status("failed", error=f"{type(e).__name__}: {e}") return False + end = log_end("load_egeria_report_specs (find_collections ReportType)", paged) + # Keep only entries whose qualified name contains "ReportType::" + report_types: list = [r for r in paged.items if isinstance(r, dict) + and "ReportType::" in (r.get("properties") or {}).get("qualifiedName", "")] + + def _settle_load() -> None: + """Mark the load done only on a confirmed end; otherwise keep what was read and retry soon, so an + unconfirmed (or filtered-empty) read is never cached for a day as the whole set.""" + global _EGERIA_SPECS_LOADED_AT, _EGERIA_SPECS_RETRY_AT, _EGERIA_SPECS_RETRY_DELAY + if paged.complete: + _EGERIA_SPECS_LOADED_AT, _EGERIA_SPECS_RETRY_AT = datetime.now(timezone.utc), None + _EGERIA_SPECS_RETRY_DELAY = None # a confirmed load resets the backoff + else: + # back off: 300 s, doubling after each unconfirmed read, capped at the cache TTL + _EGERIA_SPECS_RETRY_DELAY = min( + (_EGERIA_SPECS_RETRY_DELAY * 2) if _EGERIA_SPECS_RETRY_DELAY else _EGERIA_SPECS_RETRY_SECONDS, + max(effective_ttl, _EGERIA_SPECS_RETRY_SECONDS)) + _EGERIA_SPECS_RETRY_AT = datetime.now(timezone.utc) + timedelta(seconds=_EGERIA_SPECS_RETRY_DELAY) + logger.info(f"load_egeria_report_specs: read {len(report_types)} ReportType(s) without Egeria " + f"confirming there are no more ({end}); not marked loaded, retrying after " + f"{_EGERIA_SPECS_RETRY_AT.isoformat()}") + _set_load_status("loaded" if paged.complete else "registered_unconfirmed", count=len(report_types)) if not report_types: - logger.debug("load_egeria_report_specs: no ReportType entities found in Egeria") - _EGERIA_SPECS_LOADED_AT = datetime.now(timezone.utc) - return True + if paged.complete: + logger.debug("load_egeria_report_specs: no ReportType entities found in Egeria") + else: + logger.debug("load_egeria_report_specs: no ReportType found in what Egeria returned") + _settle_load() + return paged.complete registry = get_report_registry() @@ -4988,8 +5052,9 @@ def load_egeria_report_specs( except Exception as e: logger.warning(f"load_egeria_report_specs: error processing ReportType — skipping: {e}") - _EGERIA_SPECS_LOADED_AT = datetime.now(timezone.utc) - logger.debug(f"load_egeria_report_specs: registry updated, cache set at {_EGERIA_SPECS_LOADED_AT.isoformat()}") + _settle_load() + logger.debug(f"load_egeria_report_specs: registry updated (loaded at {_EGERIA_SPECS_LOADED_AT}, " + f"retry at {_EGERIA_SPECS_RETRY_AT})") _log_spec_resolution("Egeria archive load", force=True) return True diff --git a/pyegeria/view/dashboard_resolver.py b/pyegeria/view/dashboard_resolver.py index 110eca92..3198e131 100644 --- a/pyegeria/view/dashboard_resolver.py +++ b/pyegeria/view/dashboard_resolver.py @@ -141,9 +141,29 @@ def is_not_visible_error(exc: BaseException) -> bool: return isinstance(exc, PyegeriaUnauthorizedException) and getattr(exc, "related_http_code", None) == 403 +def is_server_not_available_error(exc: BaseException) -> bool: + """Egeria's OMAG-MULTI-TENANT-404-*: the server (or a service on it) isn't available. A 404 by number, but + a fact about the platform, never about the element (pyegeria raises it as an API error, not not-found).""" + info = getattr(exc, "additional_info", None) + message_id = str(info.get("exceptionErrorMessageId") or "") if isinstance(info, dict) else "" + try: + text = str(exc) + except Exception: # noqa: BLE001 + text = "" + return message_id.startswith("OMAG-MULTI-TENANT-404") or "OMAG-MULTI-TENANT-404" in text + + def _is_not_found_error(exc: BaseException) -> bool: + """Egeria's specific element not-found. Never OMAG-MULTI-TENANT-404 (server not available), whose text also + carries "-404-".""" from pyegeria.core._exceptions import PyegeriaNotFoundException - return isinstance(exc, PyegeriaNotFoundException) or "-404-" in str(exc) + if is_server_not_available_error(exc): + return False + try: + text = str(exc) + except Exception: # noqa: BLE001 + text = "" + return isinstance(exc, PyegeriaNotFoundException) or "-404-" in text def placements_for_perspective(placements: List[Any], perspective: Optional[str]) -> List[Any]: @@ -417,7 +437,19 @@ async def async_resolve_sheet(client: Any, sheet: Any, *, perspective: Optional[ check_spec_visibility: bool = True, fetch_missing: bool = True) -> ResolvedSheet: """Resolve a sheet the caller has ALREADY read from Egeria (an EgeriaDashboardSheet, e.g. from its own cache of load_dashboard_sheet), so an app that caches sheet reads does not walk the chain twice. The - sheet must have been read as the same viewer as `client`. Otherwise as async_resolve_dashboard_sheet.""" + sheet must have been read as the same viewer as `client`. Otherwise as async_resolve_dashboard_sheet. + + The EGERIA tier is read for the viewer's credential identity (spec-cache identity ruling, item 3): a private + cache bound to someone else isn't loaded.""" + from pyegeria.view.egeria_spec_cache import client_credential_identity, spec_identity + with spec_identity(client_credential_identity(client)): + return await _async_resolve_sheet(client, sheet, perspective=perspective, run=run, spec_lookup=spec_lookup, + check_spec_visibility=check_spec_visibility, fetch_missing=fetch_missing) + + +async def _async_resolve_sheet(client: Any, sheet: Any, *, perspective: Optional[str], run: bool, + spec_lookup: Optional[Callable], check_spec_visibility: bool, + fetch_missing: bool) -> ResolvedSheet: resolved = ResolvedSheet(name=sheet.name, heading=sheet.heading, description=sheet.description or "", family=sheet.family, aliases=list(sheet.aliases or []), guid=getattr(sheet, "guid", None), breaks=list(getattr(sheet, "breaks", None) or []), perspective=perspective, diff --git a/pyegeria/view/dashboards.py b/pyegeria/view/dashboards.py index d584cc72..00490b26 100644 --- a/pyegeria/view/dashboards.py +++ b/pyegeria/view/dashboards.py @@ -275,34 +275,30 @@ async def async_load_dashboard_sheet(client: Any, name: str) -> Optional[EgeriaD return await read_sheet_from_chain(client, name) -async def _dashboard_elements(client: Any) -> List[Dict[str, Any]]: - """Every Dashboard whose qualifiedName starts `Dashboard::`. Read by type in pages of 25 (Dashboard is a - small type; only an empty page ends the read) and filtered here: a starts-with search on qualifiedName - took 21 s on the live platform with no Dashboards at all, the type read 2.8 s.""" - out: List[Dict[str, Any]] = [] - seen: set = set() - start = 0 - while True: - page = await client.classification_manager._async_get_elements( - "Dashboard", start_from=start, page_size=PAGE_SIZE, graph_query_depth=0) - items = [e for e in page if isinstance(e, dict) and e.get("elementHeader")] if isinstance(page, list) else [] - if not items: - return out - fresh = [e for e in items if (e.get("elementHeader") or {}).get("guid") not in seen] - if not fresh: # a page of elements already read: paging did not advance, and would never end (6.2.36) - logger.warning(f"dashboards: paging did not advance at position {start}; stopped with {len(seen)} read") - return out - seen.update((e.get("elementHeader") or {}).get("guid") for e in fresh) - items = fresh - out.extend(e for e in items - if ((e.get("properties") or {}).get("qualifiedName") or "").startswith(DASHBOARD_QN_PREFIX)) - start += len(items) - - -async def _all_sheets(client: Any) -> List[EgeriaDashboardSheet]: +async def _dashboard_elements(client: Any, notes: Optional[List[str]] = None) -> List[Dict[str, Any]]: + """Every Dashboard whose qualifiedName starts `Dashboard::`. Read by type through the pager at PAGE_SIZE + (pager step 4; Dashboard is a small type) and filtered here: a starts-with search on qualifiedName + took 21 s on the live platform with no Dashboards at all, the type read 2.8 s. When the read isn't + confirmed complete, `notes` (if given) gets "Showing at least N Dashboards: ...".""" + from pyegeria.core.paging import async_page_all, async_paging_for, at_least_note, log_end + + async def fetch(start, size): + return await client.classification_manager._async_get_elements( + "Dashboard", start_from=start, page_size=size, graph_query_depth=0) + + paged = await async_page_all(fetch, fetch_page_size=PAGE_SIZE, **await async_paging_for("get_elements", client)) + log_end("dashboards._dashboard_elements", paged) + out = [e for e in paged.items if isinstance(e, dict) and e.get("elementHeader") + and ((e.get("properties") or {}).get("qualifiedName") or "").startswith(DASHBOARD_QN_PREFIX)] + if notes is not None and (words := at_least_note(paged, "dashboard sheets", len(out))): + notes.append(words) + return out + + +async def _all_sheets(client: Any, notes: Optional[List[str]] = None) -> List[EgeriaDashboardSheet]: sheets: List[EgeriaDashboardSheet] = [] seen = set() - for element in await _dashboard_elements(client): + for element in await _dashboard_elements(client, notes): name = sheet_name_from_dashboard_qn((element.get("properties") or {}).get("qualifiedName") or "") if not name or name in seen: continue @@ -410,7 +406,8 @@ async def async_list_dashboard_sheets(client: Any, folders: Optional[List[str]] (for example one made with plain `Create Dashboard`) is listed with its `breaks`, not dropped.""" chosen = configured_dashboard_folders() if folders is None else [f for f in folders if f] if not chosen: - return DashboardSheetListing(await _all_sheets(client), notes=[NO_FOLDERS_NOTE]) + listing_notes = [NO_FOLDERS_NOTE] + return DashboardSheetListing(await _all_sheets(client, listing_notes), notes=listing_notes) found, notes, status = await asyncio.to_thread(_walk_dashboard_folders, client, chosen, depth_cap) sheets: List[EgeriaDashboardSheet] = [] for name in sorted(found, key=str.lower): diff --git a/pyegeria/view/dr_egeria_reports.py b/pyegeria/view/dr_egeria_reports.py index 70151e1b..f2200ed6 100644 --- a/pyegeria/view/dr_egeria_reports.py +++ b/pyegeria/view/dr_egeria_reports.py @@ -218,7 +218,7 @@ 'Regulation-DrE-Basic': FormatSet(target_type='Regulation', heading='Regulation-DrE-Basic Attributes', description='Auto-generated format for Regulation (Create, Basic).', family='Governance Officer', formats=[Format(types=['LIST'], attributes=[Column(name='Display Name', key='display_name'), Column(name='Qualified Name', key='qualified_name'), Column(name='GUID', key='guid'), Column(name='Zone Membership', key='zone_membership'), Column(name='Description', key='description'), Column(name='Legal', key='legal'), Column(name='Authors', key='authors'), Column(name='Domain Identifier', key='domain_identifier', valid_values=['All Domains', 'Data', 'Privacy', 'Security', 'IT Infrastructure', 'Software Development', 'Corporate', 'Asset Management', 'Other']), Column(name='Implications', key='implications'), Column(name='Importance', key='importance'), Column(name='Outcomes', key='outcomes'), Column(name='Results', key='results'), Column(name='Scope', key='scope'), Column(name='Summary', key='summary'), Column(name='Usage', key='usage'), Column(name='Regulation Source', key='regulation_source'), Column(name='Regulators', key='regulators'), Column(name='Version Identifier', key='version_identifier'), Column(name='Content Status', key='content_status', valid_values=['DRAFT', 'PREPARED', 'PROPOSED', 'APPROVED', 'REJECTED', 'ACTIVE', 'DEPRECATED', 'OTHER']), Column(name='Category', key='category')]), Format(types=['ALL'], attributes=[Column(name='Zone Membership', key='zone_membership'), Column(name='Category', key='category'), Column(name='Description', key='description'), Column(name='Display Name', key='display_name'), Column(name='GUID', key='guid'), Column(name='Legal', key='legal'), Column(name='Qualified Name', key='qualified_name'), Column(name='URL', key='url'), Column(name='Version Identifier', key='version_identifier'), Column(name='Authors', key='authors'), Column(name='Content Status', key='content_status', valid_values=['DRAFT', 'PREPARED', 'PROPOSED', 'APPROVED', 'REJECTED', 'ACTIVE', 'DEPRECATED', 'OTHER']), Column(name='Domain Identifier', key='domain_identifier', valid_values=['All Domains', 'Data', 'Privacy', 'Security', 'IT Infrastructure', 'Software Development', 'Corporate', 'Asset Management', 'Other']), Column(name='Implications', key='implications'), Column(name='Importance', key='importance'), Column(name='Outcomes', key='outcomes'), Column(name='Results', key='results'), Column(name='Scope', key='scope'), Column(name='Summary', key='summary'), Column(name='Usage', key='usage'), Column(name='Regulation Source', key='regulation_source'), Column(name='Regulators', key='regulators')])], action=ActionParameter(function='GovernanceOfficer.find_governance_definitions', required_params=['search_string'], optional_params=['sequencing_order', 'sequencing_property', 'page_size', 'start_from', 'starts_with', 'ends_with', 'ignore_case', 'classification_names', 'metadata_element_subtypes', 'metadata_element_type'], spec_params={'metadata_element_type': 'Regulation'})), 'Related-Media-DrE-Advanced': FormatSet(target_type='Related Media', heading='Related-Media-DrE-Advanced Attributes', description='Auto-generated format for Related Media (Create, Advanced).', family='External Reference', formats=[Format(types=['LIST'], attributes=[Column(name='Display Name', key='display_name'), Column(name='Qualified Name', key='qualified_name'), Column(name='GUID', key='guid'), Column(name='Effective From', key='effective_from'), Column(name='Effective Time', key='effective_time'), Column(name='Effective To', key='effective_to'), Column(name='Glossary Term', key='glossary_term'), Column(name='Is Own Anchor', key='is_own_anchor'), Column(name='Zone Membership', key='zone_membership'), Column(name='Additional Properties', key='additional_properties'), Column(name='Confidence Classification', key='confidence_classification', valid_values=['UNCLASSIFIED', 'AD_HOC', 'TRANSACTIONAL', 'AUTHORITATIVE', 'DERIVED', 'OBSOLETE', 'OTHER']), Column(name='Confidentiality Classification', key='confidentiality_classification', valid_values=['UNCLASSIFIED', 'INTERNAL', 'CONFIDENTIAL', 'SENSITIVE', 'RESTRICTED', 'OTHER']), Column(name='Criticality Classification', key='criticality_classification', valid_values=['UNCLASSIFIED', 'MARGINAL', 'IMPORTANT', 'CRITICAL', 'CATASTROPHIC', 'OTHER']), Column(name='Description', key='description'), Column(name='Identifier', key='identifier'), Column(name='Impact Classification', key='impact_classification', valid_values=['UNCLASSIFIED', 'LOW', 'MEDIUM', 'HIGH', 'OTHER']), Column(name='Legal', key='legal'), Column(name='Policy Management Point', key='policy_management_point'), Column(name='Retention Classification', key='retention_classification', valid_values=['UNCLASSIFIED', 'TEMPORARY', 'PROJECT_LIFETIME', 'TEAM_LIFETIME', 'CONTRACT_LIFETIME', 'REGULATED_LIFETIME', 'TIMEBOXED_LIFETIME', 'OTHER']), Column(name='Security Tags', key='security_tags'), Column(name='Authors', key='authors'), Column(name='Attribution', key='attribution'), Column(name='Copyright', key='copyright'), Column(name='License', key='license'), Column(name='Organization', key='organization'), Column(name='Reference Abstract', key='reference_abstract'), Column(name='Reference Title', key='reference_title'), Column(name='Sources', key='reference_sources'), Column(name='Default Media Usage', key='default_media_usage', valid_values=['ICON', 'THUMBNAIL', 'ILLUSTRATION', 'USAGE_GUIDANCE', 'OTHER']), Column(name='Default Media Usage Other Id', key='default_media_usage_other_id'), Column(name='Media Type', key='media_type', valid_values=['IMAGE', 'AUDIO', 'DOCUMENT', 'VIDEO', 'OTHER']), Column(name='Media Type Other Id', key='media_type_other_id'), Column(name='Classifications', key='classifications'), Column(name='Anchor Scope Name', key='anchor_scope_guid'), Column(name='Supplementary Properties', key='supplementary_properties'), Column(name='Version Identifier', key='version_identifier'), Column(name='Status', key='status'), Column(name='Content Status', key='content_status', valid_values=['DRAFT', 'PREPARED', 'PROPOSED', 'APPROVED', 'REJECTED', 'ACTIVE', 'DEPRECATED', 'OTHER']), Column(name='User Defined Content Status', key='user_defined_content_status'), Column(name='User Defined Status', key='user_defined_status'), Column(name='Category', key='category')]), Format(types=['ALL'], attributes=[Column(name='Effective From', key='effective_from'), Column(name='Effective Time', key='effective_time'), Column(name='Effective To', key='effective_to'), Column(name='Glossary Term', key='glossary_term'), Column(name='Is Own Anchor', key='is_own_anchor'), Column(name='Status', key='status'), Column(name='Zone Membership', key='zone_membership'), Column(name='Additional Properties', key='additional_properties'), Column(name='Category', key='category'), Column(name='Confidence Classification', key='confidence_classification', valid_values=['UNCLASSIFIED', 'AD_HOC', 'TRANSACTIONAL', 'AUTHORITATIVE', 'DERIVED', 'OBSOLETE', 'OTHER']), Column(name='Confidentiality Classification', key='confidentiality_classification', valid_values=['UNCLASSIFIED', 'INTERNAL', 'CONFIDENTIAL', 'SENSITIVE', 'RESTRICTED', 'OTHER']), Column(name='Criticality Classification', key='criticality_classification', valid_values=['UNCLASSIFIED', 'MARGINAL', 'IMPORTANT', 'CRITICAL', 'CATASTROPHIC', 'OTHER']), Column(name='Description', key='description'), Column(name='Display Name', key='display_name'), Column(name='GUID', key='guid'), Column(name='Identifier', key='identifier'), Column(name='Impact Classification', key='impact_classification', valid_values=['UNCLASSIFIED', 'LOW', 'MEDIUM', 'HIGH', 'OTHER']), Column(name='Legal', key='legal'), Column(name='Policy Management Point', key='policy_management_point'), Column(name='Qualified Name', key='qualified_name'), Column(name='Retention Classification', key='retention_classification', valid_values=['UNCLASSIFIED', 'TEMPORARY', 'PROJECT_LIFETIME', 'TEAM_LIFETIME', 'CONTRACT_LIFETIME', 'REGULATED_LIFETIME', 'TIMEBOXED_LIFETIME', 'OTHER']), Column(name='Security Tags', key='security_tags'), Column(name='URL', key='url'), Column(name='Version Identifier', key='version_identifier'), Column(name='Authors', key='authors'), Column(name='Content Status', key='content_status', valid_values=['DRAFT', 'PREPARED', 'PROPOSED', 'APPROVED', 'REJECTED', 'ACTIVE', 'DEPRECATED', 'OTHER']), Column(name='User Defined Content Status', key='user_defined_content_status'), Column(name='Attribution', key='attribution'), Column(name='Copyright', key='copyright'), Column(name='License', key='license'), Column(name='Organization', key='organization'), Column(name='Reference Abstract', key='reference_abstract'), Column(name='Reference Title', key='reference_title'), Column(name='Sources', key='reference_sources'), Column(name='Default Media Usage', key='default_media_usage', valid_values=['ICON', 'THUMBNAIL', 'ILLUSTRATION', 'USAGE_GUIDANCE', 'OTHER']), Column(name='Default Media Usage Other Id', key='default_media_usage_other_id'), Column(name='Media Type', key='media_type', valid_values=['IMAGE', 'AUDIO', 'DOCUMENT', 'VIDEO', 'OTHER']), Column(name='Media Type Other Id', key='media_type_other_id'), Column(name='User Defined Status', key='user_defined_status'), Column(name='Classifications', key='classifications'), Column(name='Anchor Scope Name', key='anchor_scope_guid'), Column(name='Supplementary Properties', key='supplementary_properties')])], action=ActionParameter(function='ExternalReferences.find_external_references', required_params=['search_string'], optional_params=['sequencing_order', 'sequencing_property', 'page_size', 'start_from', 'starts_with', 'ends_with', 'ignore_case', 'classification_names', 'metadata_element_subtypes', 'metadata_element_type'], spec_params={'metadata_element_type': 'RelatedMedia'})), 'Related-Media-DrE-Basic': FormatSet(target_type='Related Media', heading='Related-Media-DrE-Basic Attributes', description='Auto-generated format for Related Media (Create, Basic).', family='External Reference', formats=[Format(types=['LIST'], attributes=[Column(name='Display Name', key='display_name'), Column(name='Qualified Name', key='qualified_name'), Column(name='GUID', key='guid'), Column(name='Zone Membership', key='zone_membership'), Column(name='Description', key='description'), Column(name='Legal', key='legal'), Column(name='Authors', key='authors'), Column(name='Attribution', key='attribution'), Column(name='Copyright', key='copyright'), Column(name='License', key='license'), Column(name='Organization', key='organization'), Column(name='Reference Abstract', key='reference_abstract'), Column(name='Reference Title', key='reference_title'), Column(name='Sources', key='reference_sources'), Column(name='Default Media Usage', key='default_media_usage', valid_values=['ICON', 'THUMBNAIL', 'ILLUSTRATION', 'USAGE_GUIDANCE', 'OTHER']), Column(name='Default Media Usage Other Id', key='default_media_usage_other_id'), Column(name='Media Type', key='media_type', valid_values=['IMAGE', 'AUDIO', 'DOCUMENT', 'VIDEO', 'OTHER']), Column(name='Media Type Other Id', key='media_type_other_id'), Column(name='Version Identifier', key='version_identifier'), Column(name='Content Status', key='content_status', valid_values=['DRAFT', 'PREPARED', 'PROPOSED', 'APPROVED', 'REJECTED', 'ACTIVE', 'DEPRECATED', 'OTHER']), Column(name='Category', key='category')]), Format(types=['ALL'], attributes=[Column(name='Zone Membership', key='zone_membership'), Column(name='Category', key='category'), Column(name='Description', key='description'), Column(name='Display Name', key='display_name'), Column(name='GUID', key='guid'), Column(name='Legal', key='legal'), Column(name='Qualified Name', key='qualified_name'), Column(name='URL', key='url'), Column(name='Version Identifier', key='version_identifier'), Column(name='Authors', key='authors'), Column(name='Content Status', key='content_status', valid_values=['DRAFT', 'PREPARED', 'PROPOSED', 'APPROVED', 'REJECTED', 'ACTIVE', 'DEPRECATED', 'OTHER']), Column(name='Attribution', key='attribution'), Column(name='Copyright', key='copyright'), Column(name='License', key='license'), Column(name='Organization', key='organization'), Column(name='Reference Abstract', key='reference_abstract'), Column(name='Reference Title', key='reference_title'), Column(name='Sources', key='reference_sources'), Column(name='Default Media Usage', key='default_media_usage', valid_values=['ICON', 'THUMBNAIL', 'ILLUSTRATION', 'USAGE_GUIDANCE', 'OTHER']), Column(name='Default Media Usage Other Id', key='default_media_usage_other_id'), Column(name='Media Type', key='media_type', valid_values=['IMAGE', 'AUDIO', 'DOCUMENT', 'VIDEO', 'OTHER']), Column(name='Media Type Other Id', key='media_type_other_id')])], action=ActionParameter(function='ExternalReferences.find_external_references', required_params=['search_string'], optional_params=['sequencing_order', 'sequencing_property', 'page_size', 'start_from', 'starts_with', 'ends_with', 'ignore_case', 'classification_names', 'metadata_element_subtypes', 'metadata_element_type'], spec_params={'metadata_element_type': 'RelatedMedia'})), - 'Report-DrE-Advanced': FormatSet(target_type='Report', heading='Report-DrE-Advanced Attributes', description='Auto-generated format for Report (Create, Advanced).', family='Views & Reports', formats=[Format(types=['LIST'], attributes=[Column(name='Display Name', key='display_name'), Column(name='Qualified Name', key='qualified_name'), Column(name='GUID', key='guid'), Column(name='Effective From', key='effective_from'), Column(name='Effective Time', key='effective_time'), Column(name='Effective To', key='effective_to'), Column(name='Glossary Term', key='glossary_term'), Column(name='Is Own Anchor', key='is_own_anchor'), Column(name='Zone Membership', key='zone_membership'), Column(name='Additional Properties', key='additional_properties'), Column(name='Confidence Classification', key='confidence_classification', valid_values=['UNCLASSIFIED', 'AD_HOC', 'TRANSACTIONAL', 'AUTHORITATIVE', 'DERIVED', 'OBSOLETE', 'OTHER']), Column(name='Confidentiality Classification', key='confidentiality_classification', valid_values=['UNCLASSIFIED', 'INTERNAL', 'CONFIDENTIAL', 'SENSITIVE', 'RESTRICTED', 'OTHER']), Column(name='Criticality Classification', key='criticality_classification', valid_values=['UNCLASSIFIED', 'MARGINAL', 'IMPORTANT', 'CRITICAL', 'CATASTROPHIC', 'OTHER']), Column(name='Description', key='description'), Column(name='Identifier', key='identifier'), Column(name='Impact Classification', key='impact_classification', valid_values=['UNCLASSIFIED', 'LOW', 'MEDIUM', 'HIGH', 'OTHER']), Column(name='Legal', key='legal'), Column(name='Policy Management Point', key='policy_management_point'), Column(name='Retention Classification', key='retention_classification', valid_values=['UNCLASSIFIED', 'TEMPORARY', 'PROJECT_LIFETIME', 'TEAM_LIFETIME', 'CONTRACT_LIFETIME', 'REGULATED_LIFETIME', 'TIMEBOXED_LIFETIME', 'OTHER']), Column(name='Security Tags', key='security_tags'), Column(name='Deployed Implementation Type', key='deployed_impl_type'), Column(name='Namespace Path', key='namespace_path'), Column(name='Resource Name', key='resource_name'), Column(name='Source', key='source'), Column(name='Analytic Parameters', key='analytic_parameters'), Column(name='AsOfTime', key='as_of_time'), Column(name='Ends With', key='ends_with'), Column(name='Governance Zone Filter', key='governance_zone_filter'), Column(name='Graph Query Depth', key='graph_query_depth'), Column(name='Ignore Case', key='ignore_case'), Column(name='Include Only Classified Elements', key='include_only_classified_elements'), Column(name='Include Only Relationships', key='include_only_relationships'), Column(name='Metadata Element Subtype Names', key='metadata_element_subtype_names'), Column(name='Metadata Element Type Name', key='metadata_element_type_name'), Column(name='Order Property Name', key='order_property_name'), Column(name='Output Format', key='output_format', valid_values=['BAR', 'DICT', 'FORM', 'GRAPH', 'HTML', 'JSON', 'KPI', 'LIST', 'MD', 'MERMAID', 'PIE', 'REPORT', 'REPORT-GRAPH', 'SERIES', 'TABLE']), Column(name='Output Sort Order', key='sort_order', valid_values=['ANY', 'CREATION_DATE_RECENT', 'CREATION_DATA_OLDEST', 'LAST_UPDATE_RECENT', 'LAST_UPDATE_OLDEST', 'PROPERTY_ASCENDING', 'PROPERTY_DESCENDING']), Column(name='Page Size', key='page_size'), Column(name='Report Parameters', key='report_parameters'), Column(name='Report Spec', key='report_spec'), Column(name='Search String', key='search_string'), Column(name='Skip Classified Elements', key='skip_classified_elements'), Column(name='Skip Relationships', key='skip_relationships'), Column(name='Start From', key='start_from'), Column(name='Starts With', key='starts_with'), Column(name='Classifications', key='classifications'), Column(name='Anchor Scope Name', key='anchor_scope_guid'), Column(name='Supplementary Properties', key='supplementary_properties'), Column(name='Version Identifier', key='version_identifier'), Column(name='Status', key='status'), Column(name='Limit Result by Status', key='limit_result_by_status', valid_values=['UNKNOWN', 'DRAFT', 'PREPARED', 'PROPOSED', 'APPROVED', 'REJECTED', 'APPROVED_CONCEPT', 'UNDER_DEVELOPMENT', 'DEVELOPMENT_COMPLETE', 'APPROVED_FOR_DEPLOYMENT', 'STANDBY', 'ACTIVE', 'FAILED', 'DISABLED', 'COMPLETE', 'DEPRECATED', 'OTHER', 'DELETED']), Column(name='User Defined Status', key='user_defined_status'), Column(name='Category', key='category')]), Format(types=['ALL'], attributes=[Column(name='Effective From', key='effective_from'), Column(name='Effective Time', key='effective_time'), Column(name='Effective To', key='effective_to'), Column(name='Glossary Term', key='glossary_term'), Column(name='Is Own Anchor', key='is_own_anchor'), Column(name='Status', key='status'), Column(name='Zone Membership', key='zone_membership'), Column(name='Additional Properties', key='additional_properties'), Column(name='Category', key='category'), Column(name='Confidence Classification', key='confidence_classification', valid_values=['UNCLASSIFIED', 'AD_HOC', 'TRANSACTIONAL', 'AUTHORITATIVE', 'DERIVED', 'OBSOLETE', 'OTHER']), Column(name='Confidentiality Classification', key='confidentiality_classification', valid_values=['UNCLASSIFIED', 'INTERNAL', 'CONFIDENTIAL', 'SENSITIVE', 'RESTRICTED', 'OTHER']), Column(name='Criticality Classification', key='criticality_classification', valid_values=['UNCLASSIFIED', 'MARGINAL', 'IMPORTANT', 'CRITICAL', 'CATASTROPHIC', 'OTHER']), Column(name='Description', key='description'), Column(name='Display Name', key='display_name'), Column(name='GUID', key='guid'), Column(name='Identifier', key='identifier'), Column(name='Impact Classification', key='impact_classification', valid_values=['UNCLASSIFIED', 'LOW', 'MEDIUM', 'HIGH', 'OTHER']), Column(name='Legal', key='legal'), Column(name='Policy Management Point', key='policy_management_point'), Column(name='Qualified Name', key='qualified_name'), Column(name='Retention Classification', key='retention_classification', valid_values=['UNCLASSIFIED', 'TEMPORARY', 'PROJECT_LIFETIME', 'TEAM_LIFETIME', 'CONTRACT_LIFETIME', 'REGULATED_LIFETIME', 'TIMEBOXED_LIFETIME', 'OTHER']), Column(name='Security Tags', key='security_tags'), Column(name='URL', key='url'), Column(name='Version Identifier', key='version_identifier'), Column(name='Deployed Implementation Type', key='deployed_impl_type'), Column(name='Namespace Path', key='namespace_path'), Column(name='Resource Name', key='resource_name'), Column(name='Source', key='source'), Column(name='Analytic Parameters', key='analytic_parameters'), Column(name='AsOfTime', key='as_of_time'), Column(name='Ends With', key='ends_with'), Column(name='Governance Zone Filter', key='governance_zone_filter'), Column(name='Graph Query Depth', key='graph_query_depth'), Column(name='Ignore Case', key='ignore_case'), Column(name='Include Only Classified Elements', key='include_only_classified_elements'), Column(name='Include Only Relationships', key='include_only_relationships'), Column(name='Limit Result by Status', key='limit_result_by_status', valid_values=['UNKNOWN', 'DRAFT', 'PREPARED', 'PROPOSED', 'APPROVED', 'REJECTED', 'APPROVED_CONCEPT', 'UNDER_DEVELOPMENT', 'DEVELOPMENT_COMPLETE', 'APPROVED_FOR_DEPLOYMENT', 'STANDBY', 'ACTIVE', 'FAILED', 'DISABLED', 'COMPLETE', 'DEPRECATED', 'OTHER', 'DELETED']), Column(name='Metadata Element Subtype Names', key='metadata_element_subtype_names'), Column(name='Metadata Element Type Name', key='metadata_element_type_name'), Column(name='Order Property Name', key='order_property_name'), Column(name='Output Format', key='output_format', valid_values=['BAR', 'DICT', 'FORM', 'GRAPH', 'HTML', 'JSON', 'KPI', 'LIST', 'MD', 'MERMAID', 'PIE', 'REPORT', 'REPORT-GRAPH', 'SERIES', 'TABLE']), Column(name='Output Sort Order', key='sort_order', valid_values=['ANY', 'CREATION_DATE_RECENT', 'CREATION_DATA_OLDEST', 'LAST_UPDATE_RECENT', 'LAST_UPDATE_OLDEST', 'PROPERTY_ASCENDING', 'PROPERTY_DESCENDING']), Column(name='Page Size', key='page_size'), Column(name='Report Parameters', key='report_parameters'), Column(name='Report Spec', key='report_spec'), Column(name='Search String', key='search_string'), Column(name='Skip Classified Elements', key='skip_classified_elements'), Column(name='Skip Relationships', key='skip_relationships'), Column(name='Start From', key='start_from'), Column(name='Starts With', key='starts_with'), Column(name='User Defined Status', key='user_defined_status'), Column(name='Classifications', key='classifications'), Column(name='Anchor Scope Name', key='anchor_scope_guid'), Column(name='Supplementary Properties', key='supplementary_properties')])], action=ActionParameter(function='ClassificationExplorer.find_elements_by_property_value', required_params=['search_string'], optional_params=['sequencing_order', 'sequencing_property', 'page_size', 'start_from', 'starts_with', 'ends_with', 'ignore_case', 'classification_names', 'metadata_element_subtypes', 'metadata_element_type'], spec_params={'metadata_element_type': 'Report', 'property_names': ['displayName', 'qualifiedName']})), + 'Report-DrE-Advanced': FormatSet(target_type='Report', heading='Report-DrE-Advanced Attributes', description='Auto-generated format for Report (Create, Advanced).', family='Views & Reports', formats=[Format(types=['LIST'], attributes=[Column(name='Display Name', key='display_name'), Column(name='Qualified Name', key='qualified_name'), Column(name='GUID', key='guid'), Column(name='Effective From', key='effective_from'), Column(name='Effective Time', key='effective_time'), Column(name='Effective To', key='effective_to'), Column(name='Glossary Term', key='glossary_term'), Column(name='Is Own Anchor', key='is_own_anchor'), Column(name='Zone Membership', key='zone_membership'), Column(name='Additional Properties', key='additional_properties'), Column(name='Confidence Classification', key='confidence_classification', valid_values=['UNCLASSIFIED', 'AD_HOC', 'TRANSACTIONAL', 'AUTHORITATIVE', 'DERIVED', 'OBSOLETE', 'OTHER']), Column(name='Confidentiality Classification', key='confidentiality_classification', valid_values=['UNCLASSIFIED', 'INTERNAL', 'CONFIDENTIAL', 'SENSITIVE', 'RESTRICTED', 'OTHER']), Column(name='Criticality Classification', key='criticality_classification', valid_values=['UNCLASSIFIED', 'MARGINAL', 'IMPORTANT', 'CRITICAL', 'CATASTROPHIC', 'OTHER']), Column(name='Description', key='description'), Column(name='Identifier', key='identifier'), Column(name='Impact Classification', key='impact_classification', valid_values=['UNCLASSIFIED', 'LOW', 'MEDIUM', 'HIGH', 'OTHER']), Column(name='Legal', key='legal'), Column(name='Policy Management Point', key='policy_management_point'), Column(name='Retention Classification', key='retention_classification', valid_values=['UNCLASSIFIED', 'TEMPORARY', 'PROJECT_LIFETIME', 'TEAM_LIFETIME', 'CONTRACT_LIFETIME', 'REGULATED_LIFETIME', 'TIMEBOXED_LIFETIME', 'OTHER']), Column(name='Security Tags', key='security_tags'), Column(name='Deployed Implementation Type', key='deployed_impl_type'), Column(name='Namespace Path', key='namespace_path'), Column(name='Resource Name', key='resource_name'), Column(name='Source', key='source'), Column(name='Analytic Parameters', key='analytic_parameters'), Column(name='AsOfTime', key='as_of_time'), Column(name='Ends With', key='ends_with'), Column(name='Governance Zone Filter', key='governance_zone_filter'), Column(name='Graph Query Depth', key='graph_query_depth'), Column(name='Ignore Case', key='ignore_case'), Column(name='Include Only Classified Elements', key='include_only_classified_elements'), Column(name='Include Only Relationships', key='include_only_relationships'), Column(name='Metadata Element Subtype Names', key='metadata_element_subtype_names'), Column(name='Metadata Element Type Name', key='metadata_element_type_name'), Column(name='Order Property Name', key='order_property_name'), Column(name='Output Format', key='output_format', valid_values=['BAR', 'DICT', 'FORM', 'GRAPH', 'HTML', 'JSON', 'KPI', 'LIST', 'MD', 'MERMAID', 'PIE', 'REPORT', 'REPORT-GRAPH', 'SERIES', 'TABLE']), Column(name='Output Sort Order', key='sort_order', valid_values=['ANY', 'CREATION_DATE_RECENT', 'CREATION_DATA_OLDEST', 'LAST_UPDATE_RECENT', 'LAST_UPDATE_OLDEST', 'PROPERTY_ASCENDING', 'PROPERTY_DESCENDING']), Column(name='Page Size', key='page_size'), Column(name='Report Parameters', key='report_parameters'), Column(name='Report Spec', key='report_spec'), Column(name='Search String', key='search_string'), Column(name='Skip Classified Elements', key='skip_classified_elements'), Column(name='Skip Relationships', key='skip_relationships'), Column(name='Start From', key='start_from'), Column(name='Starts With', key='starts_with'), Column(name='Classifications', key='classifications'), Column(name='Anchor Scope Name', key='anchor_scope_guid'), Column(name='Supplementary Properties', key='supplementary_properties'), Column(name='Version Identifier', key='version_identifier'), Column(name='Status', key='status'), Column(name='Limit Result by Status', key='limit_result_by_status', valid_values=['UNKNOWN', 'ACTIVE', 'INVALID', 'DELETED', 'OTHER']), Column(name='User Defined Status', key='user_defined_status'), Column(name='Category', key='category')]), Format(types=['ALL'], attributes=[Column(name='Effective From', key='effective_from'), Column(name='Effective Time', key='effective_time'), Column(name='Effective To', key='effective_to'), Column(name='Glossary Term', key='glossary_term'), Column(name='Is Own Anchor', key='is_own_anchor'), Column(name='Status', key='status'), Column(name='Zone Membership', key='zone_membership'), Column(name='Additional Properties', key='additional_properties'), Column(name='Category', key='category'), Column(name='Confidence Classification', key='confidence_classification', valid_values=['UNCLASSIFIED', 'AD_HOC', 'TRANSACTIONAL', 'AUTHORITATIVE', 'DERIVED', 'OBSOLETE', 'OTHER']), Column(name='Confidentiality Classification', key='confidentiality_classification', valid_values=['UNCLASSIFIED', 'INTERNAL', 'CONFIDENTIAL', 'SENSITIVE', 'RESTRICTED', 'OTHER']), Column(name='Criticality Classification', key='criticality_classification', valid_values=['UNCLASSIFIED', 'MARGINAL', 'IMPORTANT', 'CRITICAL', 'CATASTROPHIC', 'OTHER']), Column(name='Description', key='description'), Column(name='Display Name', key='display_name'), Column(name='GUID', key='guid'), Column(name='Identifier', key='identifier'), Column(name='Impact Classification', key='impact_classification', valid_values=['UNCLASSIFIED', 'LOW', 'MEDIUM', 'HIGH', 'OTHER']), Column(name='Legal', key='legal'), Column(name='Policy Management Point', key='policy_management_point'), Column(name='Qualified Name', key='qualified_name'), Column(name='Retention Classification', key='retention_classification', valid_values=['UNCLASSIFIED', 'TEMPORARY', 'PROJECT_LIFETIME', 'TEAM_LIFETIME', 'CONTRACT_LIFETIME', 'REGULATED_LIFETIME', 'TIMEBOXED_LIFETIME', 'OTHER']), Column(name='Security Tags', key='security_tags'), Column(name='URL', key='url'), Column(name='Version Identifier', key='version_identifier'), Column(name='Deployed Implementation Type', key='deployed_impl_type'), Column(name='Namespace Path', key='namespace_path'), Column(name='Resource Name', key='resource_name'), Column(name='Source', key='source'), Column(name='Analytic Parameters', key='analytic_parameters'), Column(name='AsOfTime', key='as_of_time'), Column(name='Ends With', key='ends_with'), Column(name='Governance Zone Filter', key='governance_zone_filter'), Column(name='Graph Query Depth', key='graph_query_depth'), Column(name='Ignore Case', key='ignore_case'), Column(name='Include Only Classified Elements', key='include_only_classified_elements'), Column(name='Include Only Relationships', key='include_only_relationships'), Column(name='Limit Result by Status', key='limit_result_by_status', valid_values=['UNKNOWN', 'ACTIVE', 'INVALID', 'DELETED', 'OTHER']), Column(name='Metadata Element Subtype Names', key='metadata_element_subtype_names'), Column(name='Metadata Element Type Name', key='metadata_element_type_name'), Column(name='Order Property Name', key='order_property_name'), Column(name='Output Format', key='output_format', valid_values=['BAR', 'DICT', 'FORM', 'GRAPH', 'HTML', 'JSON', 'KPI', 'LIST', 'MD', 'MERMAID', 'PIE', 'REPORT', 'REPORT-GRAPH', 'SERIES', 'TABLE']), Column(name='Output Sort Order', key='sort_order', valid_values=['ANY', 'CREATION_DATE_RECENT', 'CREATION_DATA_OLDEST', 'LAST_UPDATE_RECENT', 'LAST_UPDATE_OLDEST', 'PROPERTY_ASCENDING', 'PROPERTY_DESCENDING']), Column(name='Page Size', key='page_size'), Column(name='Report Parameters', key='report_parameters'), Column(name='Report Spec', key='report_spec'), Column(name='Search String', key='search_string'), Column(name='Skip Classified Elements', key='skip_classified_elements'), Column(name='Skip Relationships', key='skip_relationships'), Column(name='Start From', key='start_from'), Column(name='Starts With', key='starts_with'), Column(name='User Defined Status', key='user_defined_status'), Column(name='Classifications', key='classifications'), Column(name='Anchor Scope Name', key='anchor_scope_guid'), Column(name='Supplementary Properties', key='supplementary_properties')])], action=ActionParameter(function='ClassificationExplorer.find_elements_by_property_value', required_params=['search_string'], optional_params=['sequencing_order', 'sequencing_property', 'page_size', 'start_from', 'starts_with', 'ends_with', 'ignore_case', 'classification_names', 'metadata_element_subtypes', 'metadata_element_type'], spec_params={'metadata_element_type': 'Report', 'property_names': ['displayName', 'qualifiedName']})), 'Report-DrE-Basic': FormatSet(target_type='Report', heading='Report-DrE-Basic Attributes', description='Auto-generated format for Report (Create, Basic).', family='Views & Reports', formats=[Format(types=['LIST'], attributes=[Column(name='Display Name', key='display_name'), Column(name='Qualified Name', key='qualified_name'), Column(name='GUID', key='guid'), Column(name='Zone Membership', key='zone_membership'), Column(name='Description', key='description'), Column(name='Legal', key='legal'), Column(name='Deployed Implementation Type', key='deployed_impl_type'), Column(name='Namespace Path', key='namespace_path'), Column(name='Resource Name', key='resource_name'), Column(name='Analytic Parameters', key='analytic_parameters'), Column(name='Ends With', key='ends_with'), Column(name='Ignore Case', key='ignore_case'), Column(name='Metadata Element Subtype Names', key='metadata_element_subtype_names'), Column(name='Metadata Element Type Name', key='metadata_element_type_name'), Column(name='Output Format', key='output_format', valid_values=['BAR', 'DICT', 'FORM', 'GRAPH', 'HTML', 'JSON', 'KPI', 'LIST', 'MD', 'MERMAID', 'PIE', 'REPORT', 'REPORT-GRAPH', 'SERIES', 'TABLE']), Column(name='Page Size', key='page_size'), Column(name='Report Parameters', key='report_parameters'), Column(name='Report Spec', key='report_spec'), Column(name='Search String', key='search_string'), Column(name='Start From', key='start_from'), Column(name='Starts With', key='starts_with'), Column(name='Version Identifier', key='version_identifier'), Column(name='Category', key='category')]), Format(types=['ALL'], attributes=[Column(name='Zone Membership', key='zone_membership'), Column(name='Category', key='category'), Column(name='Description', key='description'), Column(name='Display Name', key='display_name'), Column(name='GUID', key='guid'), Column(name='Legal', key='legal'), Column(name='Qualified Name', key='qualified_name'), Column(name='URL', key='url'), Column(name='Version Identifier', key='version_identifier'), Column(name='Deployed Implementation Type', key='deployed_impl_type'), Column(name='Namespace Path', key='namespace_path'), Column(name='Resource Name', key='resource_name'), Column(name='Analytic Parameters', key='analytic_parameters'), Column(name='Ends With', key='ends_with'), Column(name='Ignore Case', key='ignore_case'), Column(name='Metadata Element Subtype Names', key='metadata_element_subtype_names'), Column(name='Metadata Element Type Name', key='metadata_element_type_name'), Column(name='Output Format', key='output_format', valid_values=['BAR', 'DICT', 'FORM', 'GRAPH', 'HTML', 'JSON', 'KPI', 'LIST', 'MD', 'MERMAID', 'PIE', 'REPORT', 'REPORT-GRAPH', 'SERIES', 'TABLE']), Column(name='Page Size', key='page_size'), Column(name='Report Parameters', key='report_parameters'), Column(name='Report Spec', key='report_spec'), Column(name='Search String', key='search_string'), Column(name='Start From', key='start_from'), Column(name='Starts With', key='starts_with')])], action=ActionParameter(function='ClassificationExplorer.find_elements_by_property_value', required_params=['search_string'], optional_params=['sequencing_order', 'sequencing_property', 'page_size', 'start_from', 'starts_with', 'ends_with', 'ignore_case', 'classification_names', 'metadata_element_subtypes', 'metadata_element_type'], spec_params={'metadata_element_type': 'Report', 'property_names': ['displayName', 'qualifiedName']})), 'Report-Spec-DrE-Advanced': FormatSet(target_type='Report Spec', heading='Report-Spec-DrE-Advanced Attributes', description='Auto-generated format for Report Spec (Create, Advanced).', family='Views & Reports', formats=[Format(types=['LIST'], attributes=[Column(name='Display Name', key='display_name'), Column(name='Qualified Name', key='qualified_name'), Column(name='GUID', key='guid'), Column(name='Effective From', key='effective_from'), Column(name='Effective Time', key='effective_time'), Column(name='Effective To', key='effective_to'), Column(name='Glossary Term', key='glossary_term'), Column(name='Is Own Anchor', key='is_own_anchor'), Column(name='Zone Membership', key='zone_membership'), Column(name='Additional Properties', key='additional_properties'), Column(name='Confidence Classification', key='confidence_classification', valid_values=['UNCLASSIFIED', 'AD_HOC', 'TRANSACTIONAL', 'AUTHORITATIVE', 'DERIVED', 'OBSOLETE', 'OTHER']), Column(name='Confidentiality Classification', key='confidentiality_classification', valid_values=['UNCLASSIFIED', 'INTERNAL', 'CONFIDENTIAL', 'SENSITIVE', 'RESTRICTED', 'OTHER']), Column(name='Criticality Classification', key='criticality_classification', valid_values=['UNCLASSIFIED', 'MARGINAL', 'IMPORTANT', 'CRITICAL', 'CATASTROPHIC', 'OTHER']), Column(name='Description', key='description'), Column(name='Identifier', key='identifier'), Column(name='Impact Classification', key='impact_classification', valid_values=['UNCLASSIFIED', 'LOW', 'MEDIUM', 'HIGH', 'OTHER']), Column(name='Legal', key='legal'), Column(name='Policy Management Point', key='policy_management_point'), Column(name='Retention Classification', key='retention_classification', valid_values=['UNCLASSIFIED', 'TEMPORARY', 'PROJECT_LIFETIME', 'TEAM_LIFETIME', 'CONTRACT_LIFETIME', 'REGULATED_LIFETIME', 'TIMEBOXED_LIFETIME', 'OTHER']), Column(name='Security Tags', key='security_tags'), Column(name='Authors', key='authors'), Column(name='Purpose', key='purpose'), Column(name='Microflow', key='microflow'), Column(name='Target Type Name', key='target_type_name'), Column(name='Aliases', key='aliases'), Column(name='Output Types', key='output_types'), Column(name='Include Preamble', key='include_preamble'), Column(name='Classifications', key='classifications'), Column(name='Anchor Scope Name', key='anchor_scope_guid'), Column(name='Supplementary Properties', key='supplementary_properties'), Column(name='Version Identifier', key='version_identifier'), Column(name='Status', key='status'), Column(name='Content Status', key='content_status', valid_values=['DRAFT', 'PREPARED', 'PROPOSED', 'APPROVED', 'REJECTED', 'ACTIVE', 'DEPRECATED', 'OTHER']), Column(name='User Defined Content Status', key='user_defined_content_status'), Column(name='User Defined Status', key='user_defined_status'), Column(name='Category', key='category')]), Format(types=['ALL'], attributes=[Column(name='Effective From', key='effective_from'), Column(name='Effective Time', key='effective_time'), Column(name='Effective To', key='effective_to'), Column(name='Glossary Term', key='glossary_term'), Column(name='Is Own Anchor', key='is_own_anchor'), Column(name='Status', key='status'), Column(name='Zone Membership', key='zone_membership'), Column(name='Additional Properties', key='additional_properties'), Column(name='Category', key='category'), Column(name='Confidence Classification', key='confidence_classification', valid_values=['UNCLASSIFIED', 'AD_HOC', 'TRANSACTIONAL', 'AUTHORITATIVE', 'DERIVED', 'OBSOLETE', 'OTHER']), Column(name='Confidentiality Classification', key='confidentiality_classification', valid_values=['UNCLASSIFIED', 'INTERNAL', 'CONFIDENTIAL', 'SENSITIVE', 'RESTRICTED', 'OTHER']), Column(name='Criticality Classification', key='criticality_classification', valid_values=['UNCLASSIFIED', 'MARGINAL', 'IMPORTANT', 'CRITICAL', 'CATASTROPHIC', 'OTHER']), Column(name='Description', key='description'), Column(name='Display Name', key='display_name'), Column(name='GUID', key='guid'), Column(name='Identifier', key='identifier'), Column(name='Impact Classification', key='impact_classification', valid_values=['UNCLASSIFIED', 'LOW', 'MEDIUM', 'HIGH', 'OTHER']), Column(name='Legal', key='legal'), Column(name='Policy Management Point', key='policy_management_point'), Column(name='Qualified Name', key='qualified_name'), Column(name='Retention Classification', key='retention_classification', valid_values=['UNCLASSIFIED', 'TEMPORARY', 'PROJECT_LIFETIME', 'TEAM_LIFETIME', 'CONTRACT_LIFETIME', 'REGULATED_LIFETIME', 'TIMEBOXED_LIFETIME', 'OTHER']), Column(name='Security Tags', key='security_tags'), Column(name='URL', key='url'), Column(name='Version Identifier', key='version_identifier'), Column(name='Authors', key='authors'), Column(name='Content Status', key='content_status', valid_values=['DRAFT', 'PREPARED', 'PROPOSED', 'APPROVED', 'REJECTED', 'ACTIVE', 'DEPRECATED', 'OTHER']), Column(name='User Defined Content Status', key='user_defined_content_status'), Column(name='Purpose', key='purpose'), Column(name='Microflow', key='microflow'), Column(name='Target Type Name', key='target_type_name'), Column(name='Aliases', key='aliases'), Column(name='Output Types', key='output_types'), Column(name='Include Preamble', key='include_preamble'), Column(name='User Defined Status', key='user_defined_status'), Column(name='Classifications', key='classifications'), Column(name='Anchor Scope Name', key='anchor_scope_guid'), Column(name='Supplementary Properties', key='supplementary_properties')])], action=ActionParameter(function='CollectionManager.find_collections', required_params=['search_string'], optional_params=['sequencing_order', 'sequencing_property', 'page_size', 'start_from', 'starts_with', 'ends_with', 'ignore_case', 'classification_names', 'metadata_element_subtypes', 'metadata_element_type'], spec_params={'metadata_element_type': 'ReportSpec'})), 'Report-Spec-DrE-Basic': FormatSet(target_type='Report Spec', heading='Report-Spec-DrE-Basic Attributes', description='Auto-generated format for Report Spec (Create, Basic).', family='Views & Reports', formats=[Format(types=['LIST'], attributes=[Column(name='Display Name', key='display_name'), Column(name='Qualified Name', key='qualified_name'), Column(name='GUID', key='guid'), Column(name='Zone Membership', key='zone_membership'), Column(name='Description', key='description'), Column(name='Legal', key='legal'), Column(name='Authors', key='authors'), Column(name='Purpose', key='purpose'), Column(name='Microflow', key='microflow'), Column(name='Target Type Name', key='target_type_name'), Column(name='Aliases', key='aliases'), Column(name='Output Types', key='output_types'), Column(name='Include Preamble', key='include_preamble'), Column(name='Version Identifier', key='version_identifier'), Column(name='Content Status', key='content_status', valid_values=['DRAFT', 'PREPARED', 'PROPOSED', 'APPROVED', 'REJECTED', 'ACTIVE', 'DEPRECATED', 'OTHER']), Column(name='Category', key='category')]), Format(types=['ALL'], attributes=[Column(name='Zone Membership', key='zone_membership'), Column(name='Category', key='category'), Column(name='Description', key='description'), Column(name='Display Name', key='display_name'), Column(name='GUID', key='guid'), Column(name='Legal', key='legal'), Column(name='Qualified Name', key='qualified_name'), Column(name='URL', key='url'), Column(name='Version Identifier', key='version_identifier'), Column(name='Authors', key='authors'), Column(name='Content Status', key='content_status', valid_values=['DRAFT', 'PREPARED', 'PROPOSED', 'APPROVED', 'REJECTED', 'ACTIVE', 'DEPRECATED', 'OTHER']), Column(name='Purpose', key='purpose'), Column(name='Microflow', key='microflow'), Column(name='Target Type Name', key='target_type_name'), Column(name='Aliases', key='aliases'), Column(name='Output Types', key='output_types'), Column(name='Include Preamble', key='include_preamble')])], action=ActionParameter(function='CollectionManager.find_collections', required_params=['search_string'], optional_params=['sequencing_order', 'sequencing_property', 'page_size', 'start_from', 'starts_with', 'ends_with', 'ignore_case', 'classification_names', 'metadata_element_subtypes', 'metadata_element_type'], spec_params={'metadata_element_type': 'ReportSpec'})), diff --git a/pyegeria/view/egeria_folders.py b/pyegeria/view/egeria_folders.py index a45d9f9a..1c1fb4d9 100644 --- a/pyegeria/view/egeria_folders.py +++ b/pyegeria/view/egeria_folders.py @@ -92,25 +92,30 @@ class FolderTooLarge(RuntimeError): """A folder has more members than one walk reads; reported as a folder not fully read, never as a short one.""" -def _members(client, collection_guid: str) -> List[Dict[str, Any]]: - """Every member of a collection, each once: paged until Egeria returns an EMPTY page (a short page is not the - last), deduplicated by GUID, and stopped when a page brings nothing new (a server that ignores paging returns the - same page again; 6.2.36). It used to be one unpaged call, so a folder over the server's default page was - silently truncated (Architect, LD-3 note N2). More than MEMBERS_MAX_PAGES pages raises FolderTooLarge.""" - out: List[Dict[str, Any]] = [] - seen: set = set() - for page_no in range(MEMBERS_MAX_PAGES): - page = _call(client.get_collection_members, collection_guid=collection_guid, - start_from=page_no * MEMBERS_PAGE, page_size=MEMBERS_PAGE, output_format="JSON") - if not page: - return out - fresh = [m for m in page if (_guid(m) or id(m)) not in seen] - if not fresh: - return out - for m in fresh: - seen.add(_guid(m) or id(m)) - out.extend(fresh) - raise FolderTooLarge(f"more than {MEMBERS_MAX_PAGES * MEMBERS_PAGE} members; not read to the end") +def _members(client, collection_guid: str, unconfirmed: Optional[List[str]] = None) -> List[Dict[str, Any]]: + """Every member of a collection, each once: read through the pager (pager step 4; a suspect path), + advancing by the page size, deduplicated by GUID, and stopped when a page brings nothing new (a server + that ignores paging returns the same page again; 6.2.36). It used to be one unpaged call, so a folder over + the server's default page was silently truncated (Architect, LD-3 note N2). More than MEMBERS_MAX_PAGES + pages raises FolderTooLarge: never shown as a short folder. Any read Egeria didn't confirm complete (the + usual suspect-path end, a stall, the empty-run cap) returns what was read and appends `collection_guid` to + `unconfirmed`, so its count is shown as "at least".""" + from pyegeria.core.paging import log_end, page_all, paging_for + + def fetch(start, size): + result = client.get_collection_members(collection_guid=collection_guid, start_from=start, page_size=size, + output_format="JSON") + return _elements(result) if isinstance(result, (list, dict)) else result # null stays null + + paged = page_all(fetch, fetch_page_size=MEMBERS_PAGE, max_pages=MEMBERS_MAX_PAGES, + **paging_for("get_collection_members"), + key=lambda m: _guid(m) or id(m)) + log_end("egeria_folders._members", paged) + if paged.end_reason == "max_pages": + raise FolderTooLarge(f"more than {MEMBERS_MAX_PAGES * MEMBERS_PAGE} members; not read to the end") + if not paged.complete and unconfirmed is not None: + unconfirmed.append(collection_guid) + return list(paged.items) def client_identity(client) -> str: @@ -144,11 +149,14 @@ def is_folder(el: Dict[str, Any]) -> bool: # ── Reads ── def find_folder(client, qn: str) -> Optional[Dict[str, Any]]: - """The CollectionFolder whose qualified name is exactly `qn`, or None.""" - found = _call(client.get_collections_by_name, qn, metadata_element_type="CollectionFolder", - output_format="JSON") - exact = [e for e in found if _props(e).get("qualifiedName") == qn] - return exact[0] if exact else None + """The CollectionFolder whose qualified name is exactly `qn`, or None when Egeria says there is none. + + Egeria's exact unique-name read, not a by-name search: one page of `get_collections_by_name` + (findMetadataElements, NULL_NOT_END before #9453) could read a filtered page as "not found", and a sync + then treats the folder as gone. Raises UniqueNameUnknown (never None) when Egeria's answer is neither the + folder nor its not-found, so callers say "could not be read" and keep what the folder held.""" + from pyegeria.core.unique_name import element_by_unique_name + return element_by_unique_name(client, qn, "CollectionFolder", what="the folder") def is_gone_error(exc: BaseException) -> bool: @@ -175,6 +183,9 @@ class Walk: gone: List[Tuple[str, str]] = field(default_factory=list) # sub-folders not visible or not found #: Every non-folder member's folder paths, by GUID: where in the tree each was found. paths: Dict[str, List[str]] = field(default_factory=dict) + #: Folders whose member read reached the end without Egeria confirming it (pager step 4): their counts are + #: "at least", and "used by none" can't be ruled out against them. + unconfirmed: List[str] = field(default_factory=list) def walk_folder(client, folder: Dict[str, Any], *, recursive: bool, consumer_prefix: str, @@ -199,10 +210,10 @@ def walk_folder(client, folder: Dict[str, Any], *, recursive: bool, consumer_pre while queue: current, depth, path = queue.pop(0) if depth == 0: - listed = _members(client, _guid(current)) + listed = _members(client, _guid(current), w.unconfirmed) else: try: - listed = _members(client, _guid(current)) + listed = _members(client, _guid(current), w.unconfirmed) except Exception as exc: # noqa: BLE001 if is_gone_error(exc): # Not visible (zoned away from the reader) or deleted: what it held is no longer @@ -265,16 +276,18 @@ def read_consumer_tree(client, *, root_qn: str, consumer_prefix: str, key: Membe identity = client_identity(client) out: Dict[str, Any] = {"format": fmt, "root": root_qn, "read_at": _now(), "read_as": identity, "status": "read", "reason": None, "consumers": [], "index": {}, "notes": []} - try: - found = _call(client.get_collections_by_name, root_qn, metadata_element_type="RootCollection", - output_format="JSON") - except Exception as exc: # noqa: BLE001 - return _not_read(out, f"the root {root_qn} could not be read: {exc}") - roots = [e for e in found if _props(e).get("qualifiedName") == root_qn] - if not roots: + # The root by Egeria's exact unique-name read (not one page of a by-name search, which could read a + # filtered page as "not found"); "can't tell" is "not read", never "not found" + from pyegeria.core.unique_name import by_unique_name + root = by_unique_name(client, root_qn, "RootCollection") + if root.answer == "unknown": + return _not_read(out, f"the root {root_qn} could not be read: {root.reason}") + if not root.found: return _not_read(out, f"the root {root_qn} was not found, or is not visible to {identity}") + roots = [root.element] + unconfirmed: List[str] = [] try: - listed = _members(client, _guid(roots[0])) + listed = _members(client, _guid(roots[0]), unconfirmed) except Exception as exc: # noqa: BLE001 return _not_read(out, f"the members of the root {root_qn} could not be read: {exc}") folders = [m for m in listed if is_folder(m)] @@ -310,7 +323,14 @@ def read_consumer_tree(client, *, root_qn: str, consumer_prefix: str, key: Membe for k in keys: index.setdefault(k, []).append(name) entry[count_field] = len(keys) + if walk.unconfirmed: # pager step 4: an "at least" count, said in the entry itself + entry["end_confirmed"] = False + unconfirmed.extend(walk.unconfirmed) out["index"] = {k: sorted(v) for k, v in sorted(index.items())} + if unconfirmed: + out["end_confirmed"] = False + out["notes"].append(f"{len(unconfirmed)} folder read(s) ended without Egeria confirming there are no more, so " + f"the counts are at least what was read and 'used by none' can't be ruled out") if unread: out.update(status="partial", reason="not fully read: " + ", ".join(unread)) return out diff --git a/pyegeria/view/egeria_spec_cache.py b/pyegeria/view/egeria_spec_cache.py index 4e865a93..fe672a47 100644 --- a/pyegeria/view/egeria_spec_cache.py +++ b/pyegeria/view/egeria_spec_cache.py @@ -26,12 +26,17 @@ from __future__ import annotations +from pyegeria.core._no_password import NO_PASSWORD, carried_password, password_as_given, resolve_password import json import os import re import tempfile +import threading import time -from dataclasses import dataclass, field +from collections import OrderedDict +from contextlib import contextmanager, nullcontext +from contextvars import ContextVar +from dataclasses import dataclass, field, replace from datetime import datetime, timezone from pathlib import Path from typing import Any, Dict, Iterable, List, Optional, Tuple @@ -66,6 +71,9 @@ "async_fetch_spec_for_call", "egeria_tier", "egeria_tier_info", + "as_client", + "declare_multi_user", + "is_multi_user", "spec_footer", "fetched_spec", "fetched_spec_source", @@ -146,6 +154,150 @@ def _settings_env(): return None +# ── Whose cache: the credential identity (spec-cache identity ruling, item 3) ── +# A private manifest is bound to the credential identity that synced it, and the tier loads it only for that +# identity. The identity is the Egeria user on a platform and view server, never the OS user. It is the user id, +# not a token hash, because a manifest outlives every token: a CLI mints a new token on each run. + +from pyegeria.core._client_identities import declare_multi_user, is_multi_user # noqa: E402 -- re-exported + +class _Unset: + __slots__ = () + + def __repr__(self) -> str: + return "UNSET" + + +#: The ContextVar's default: no spec_identity block is active. Distinct from None, which a block sets to mean +#: "explicitly no one" (T4 (3)): None is never read as unset, so it never falls back to a process identity. +_UNSET: Any = _Unset() +_SPEC_IDENTITY: "ContextVar[Any]" = ContextVar("pyegeria_spec_identity", default=_UNSET) +_REFUSED_LOGGED: set = set() + + +def credential_identity(user_id: Any, platform_url: Optional[str] = None, + view_server: Optional[str] = None) -> Optional[str]: + """"@/", or None when there is no user.""" + user = user_id.strip() if isinstance(user_id, str) else "" + if not user or user == "?": + return None + env = _settings_env() + url = platform_url if isinstance(platform_url, str) and platform_url else ( + getattr(env, "egeria_view_server_url", None) if env else None) + server = view_server if isinstance(view_server, str) and view_server else ( + getattr(env, "egeria_view_server", None) if env else None) + url = url if isinstance(url, str) else None + server = server if isinstance(server, str) else None + return f"{user}@{_host_port(url)}/{_safe(server or 'unknown-view-server')}" + + +def client_credential_identity(client: Any) -> Optional[str]: + """The credential identity of the Egeria client a read is made with.""" + return credential_identity(getattr(client, "user_id", None), getattr(client, "platform_url", None), + getattr(client, "view_server", None) or getattr(client, "server_name", None)) + + +@contextmanager +def spec_identity(identity: Optional[str]): + """Run the block as `identity` for the EGERIA tier (exec_report_spec and the async runner set it). + ``spec_identity(None)`` (or an empty identity) runs it as explicitly NO ONE: no private cache loads and no + fetched copy is served, whether or not the process is declared multi-user; it never falls back.""" + token = _SPEC_IDENTITY.set(identity if identity else None) + try: + yield + finally: + _SPEC_IDENTITY.reset(token) + + +_MULTI_LOGGED: set = set() + + +def as_client(client: Any): + """`spec_identity(client_credential_identity(client))`: every EGERIA-tier read a path makes with a client in + hand is for that client's credential, never for the process fallback. A client with no readable identity + (a bare object) leaves the current identity as it is.""" + identity = client_credential_identity(client) if client is not None else None + if identity: + _note_client(client) # its current identity (after any user_id swap) counts towards "more than one" + return spec_identity(identity) if identity else nullcontext() + + +def current_spec_identity() -> Optional[str]: + """The identity the EGERIA tier is read for: + 1. the one set by `spec_identity` (exec_report_spec, the async runner and the dashboard resolver set it + from the request's client). Inside ``spec_identity(None)`` that is None, explicitly no one, with no + fallback below; + In a process DECLARED multi-user (`pyegeria.core._client_identities.declare_multi_user()` or + PYEGERIA_MULTI_USER=1: the Portal, EA, Studio, the MCP HTTP server), nothing else: None, so no private cache + loads and no fetched copy is served outside a request's own identity. Otherwise: + 2. the credentials of this process's Egeria clients (`pyegeria.core._client_identities`), when it has seen + exactly one: a single-user process (a CLI, the stdio MCP server) reads as the user its client signs in + as, whatever the settings say; + 3. when it has seen more than one, None (logged once). This is only a BACKSTOP for an undeclared server: + it can be skipped, because a server that builds clients as its service user and then swaps user_id + (the Portal's use_token) records the service user first, and reads as it until a second identity is + seen. A multi-user process must declare itself; + 4. else, with no client yet, this process's configured Egeria user, resolved exactly as a CLI's client is + (`pyegeria.core.config.configured_user_id`: EGERIA_USER, the settings' user, the quickstart default). + A path that holds a client binds to it instead (`as_client`), whatever this fallback says.""" + identity = _SPEC_IDENTITY.get() + if identity is not _UNSET: + return identity # set by a spec_identity block: a credential identity, or None for explicitly no one + try: + from pyegeria.core._client_identities import is_multi_user + if is_multi_user(): + return None + except Exception: # noqa: BLE001 + return None # can't tell: fail safe + try: + from pyegeria.core._client_identities import seen_credentials + seen = {credential_identity(*entry) for entry in seen_credentials()} - {None} + except Exception: # noqa: BLE001 + seen = set() + if len(seen) == 1: + return next(iter(seen)) + if seen: + key = frozenset(seen) + if key not in _MULTI_LOGGED: + _MULTI_LOGGED.add(key) + logger.info(f"Report spec cache: this process has Egeria clients for {len(seen)} identities; a read " + f"outside a request's own identity (spec_identity) loads no private cache") + return None + try: + from pyegeria.core.config import configured_user_id + return credential_identity(configured_user_id()) + except Exception: # noqa: BLE001 -- no settings: no identity, so no private manifest loads (fail safe) + return None + + +def _manifest_loads_for(directory: Path, manifest: Dict[str, Any], identity: Optional[str]) -> bool: + """Whether a manifest may load into the tier for `identity`. A shared one loads for anyone; a private one + only for the identity it is bound to (one without a binding, e.g. from before this rule, for no one). + A refusal is logged once per (directory, bound identity, identity).""" + if manifest.get("shared") or not manifest.get("entries"): + return True + bound = manifest.get("bound_identity") + if bound and bound == identity: + return True + key = (str(directory), bound, identity) + if key not in _REFUSED_LOGGED: + _REFUSED_LOGGED.add(key) + why = (f"it is bound to {bound}" if bound else + "it records no bound identity (synced before private caches were bound); re-sync to bind it") + logger.warning(f"Report spec cache {directory}: a private cache, not loaded for {identity or 'no identity'}: " + f"{why}") + return False + + +def _note_client(client: Any) -> None: + """Record a sync's or fetch's client as one of this process's identities (see current_spec_identity).""" + try: + from pyegeria.core._client_identities import note_client + note_client(client) + except Exception: # noqa: BLE001 + pass + + def _host_port(url: Optional[str]) -> str: if not url: return "unknown-platform" @@ -335,6 +487,28 @@ def _zones(el: Dict[str, Any]) -> List[str]: return sorted(set(found)) +def _zone_state(el: Dict[str, Any]) -> Tuple[str, List[str]]: + """("public", []), ("zoned", zones) or ("unknown", []) for an element's visibility. Unknown: there is no + element header to read, or a ZoneMembership is present whose zones can't be read (neither a list nor a + string). No zones means open (Egeria's zone rule).""" + header = _header(el) + if not isinstance(header, dict) or not header: + return "unknown", [] + candidates = [header.get("zoneMembership")] + [c for c in (header.get("classifications") or []) + if isinstance(c, dict) + and c.get("classificationName") == "ZoneMembership"] + for c in candidates: + if c is None: + continue + if not isinstance(c, dict): + return "unknown", [] + z = (c.get("classificationProperties") or {}).get("zoneMembership") + if z is not None and not isinstance(z, (list, str)): + return "unknown", [] + zones = _zones(el) + return ("zoned", zones) if zones else ("public", []) + + ZONED_REASON = "zoned: not synced into a shared cache (until B6's public-only read)" @@ -477,7 +651,8 @@ def _known_names() -> set: from pyegeria.view import base_report_formats as brf names = set(brf._builtin_tier().keys()) names |= set(brf._CONFIG_REPORT_FORMATS.keys()) | set(brf._RUNTIME_REPORT_FORMATS.keys()) - names |= set(_TIER.get("specs", {}).keys()) + entry = _peek_tier_entry() # the caller's own entry, never another identity's; peeked: this never loads + names |= set(entry.specs.keys()) if entry is not None else set() return names @@ -556,20 +731,104 @@ def _find_folder(client, qn: str) -> Optional[Dict[str, Any]]: return _folders.find_folder(client, qn) +#: The label lookup pages each by-name search through the pager (findMetadataElements underneath: NULL_NOT_END +#: before #9453, trusted on a gated platform), up to this many requests of SPEC_LOOKUP_PAGE per name. +SPEC_LOOKUP_PAGE = 100 +SPEC_LOOKUP_MAX_PAGES = 10 + + +class SpecLookupUnconfirmed(RuntimeError): + """Egeria returned no ReportSpec for a label, but didn't confirm there is none (or a lookup failed): the + words say "couldn't tell", never "not found".""" + + +def _spec_label_names(label: str) -> Tuple[str, str, str]: + return label, f"ReportSpec::{label}", f"ReportType::{label}" + + +def _spec_matches(read_items: Iterable[Any], label: str, seen: Dict[str, Dict[str, Any]]) -> None: + for e in read_items: + if isinstance(e, dict) and label_from_qualified_name(_props(e).get("qualifiedName") or "") == label \ + and _guid(e): + seen[_guid(e)] = e + + +def _spec_lookup_result(label: str, seen: Dict[str, Dict[str, Any]], unconfirmed: List[str]) -> List[Dict[str, Any]]: + """The matches; with none, "not found" only when every read reached a confirmed end.""" + if not seen and unconfirmed: + raise SpecLookupUnconfirmed(f"couldn't tell whether Egeria holds report spec '{label}': none was found in " + f"what Egeria returned, but it didn't confirm there are no more " + f"({'; '.join(unconfirmed)})") + return list(seen.values()) + + def _find_specs_by_label(client, label: str) -> List[Dict[str, Any]]: - """Every ReportSpec element whose label is `label`, under either qualified-name marker.""" + """Every ReportSpec element whose label is `label`, under either qualified-name marker. + + A label isn't an exact qualified name (a local qualifier may precede the marker), so each by-name search is + paged with `paging_for` rather than read as one page. [] only when every read reached a confirmed end; + otherwise, with no match, SpecLookupUnconfirmed (never "not found").""" + from pyegeria.core.paging import page_all, paging_for seen: Dict[str, Dict[str, Any]] = {} - for name in (label, f"ReportSpec::{label}", f"ReportType::{label}"): + unconfirmed: List[str] = [] + paging = paging_for("get_collections_by_name", client) + for name in _spec_label_names(label): + def fetch(start: int, size: int, name: str = name): + return client.get_collections_by_name(name, metadata_element_type="ReportSpec", output_format="JSON", + start_from=start, page_size=size) try: - found = _call(client.get_collections_by_name, name, metadata_element_type="ReportSpec", - output_format="JSON") + read = page_all(fetch, fetch_page_size=SPEC_LOOKUP_PAGE, max_pages=SPEC_LOOKUP_MAX_PAGES, **paging) except Exception as exc: # noqa: BLE001 -- one failing lookup must not hide the others logger.debug(f"report spec lookup by name {name!r} failed: {exc}") + unconfirmed.append(f"{name!r}: the read failed ({type(exc).__name__})") continue - for e in found: - if label_from_qualified_name(_props(e).get("qualifiedName") or "") == label and _guid(e): - seen[_guid(e)] = e - return list(seen.values()) + if not read.complete: + unconfirmed.append(f"{name!r}: {read.end_reason}") + _spec_matches(read.items, label, seen) + return _spec_lookup_result(label, seen, unconfirmed) + + +async def _async_find_specs_by_label(client, label: str) -> List[Dict[str, Any]]: + """`_find_specs_by_label` for async code.""" + from pyegeria.core.paging import async_page_all, async_paging_for + seen: Dict[str, Dict[str, Any]] = {} + unconfirmed: List[str] = [] + paging = await async_paging_for("_async_get_collections_by_name", client) + for name in _spec_label_names(label): + async def fetch(start: int, size: int, name: str = name): + return await client._async_get_collections_by_name(name, metadata_element_type="ReportSpec", + output_format="JSON", start_from=start, + page_size=size) + try: + read = await async_page_all(fetch, fetch_page_size=SPEC_LOOKUP_PAGE, max_pages=SPEC_LOOKUP_MAX_PAGES, + **paging) + except Exception as exc: # noqa: BLE001 + logger.debug(f"report spec lookup by name {name!r} failed: {exc}") + unconfirmed.append(f"{name!r}: the read failed ({type(exc).__name__})") + continue + if not read.complete: + unconfirmed.append(f"{name!r}: {read.end_reason}") + _spec_matches(read.items, label, seen) + return _spec_lookup_result(label, seen, unconfirmed) + + +def _pinned_element_state(client, guid: str) -> Tuple[str, str]: + """Whether a pinned spec's element is still there, by GUID, three ways: ("found", ""), ("gone", why) + (Egeria's specific element not-found, or the element is no longer visible to the syncing identity: the + only answers that prune), or ("unknown", the error class): any other error, OMAG-MULTI-TENANT-404 (the + server or service isn't available) included, which keeps the entry.""" + if not guid: + return "unknown", "no GUID recorded" + try: + el = _get_full(client, guid) + except Exception as exc: # noqa: BLE001 -- classified below; only an element's absence prunes + if _folders.is_gone_error(exc): + from pyegeria.view.dashboard_resolver import is_not_visible_error + if is_not_visible_error(exc): + return "gone", f"is no longer visible to {_client_identity(client)}" + return "gone", "no longer exists" + return "unknown", type(exc).__name__ + return ("found", "") if el else ("gone", "no longer exists") # NO_ELEMENTS_FOUND: Egeria's not-found def _get_full(client, guid: str) -> Optional[Dict[str, Any]]: @@ -630,7 +889,8 @@ def sync_report_specs(client, *, folders: Optional[Iterable[str]] = None, specs: report.notes.append(f"no report spec folders are configured (set {FOLDERS_ENV} or config.json's " f"\"Pyegeria Report Spec Folders\", or pass --folder/--spec); nothing synced") return report - with _SyncLock(directory): + _note_client(client) + with as_client(client), _SyncLock(directory): identity = _client_identity(client) # Gather candidates: guid -> (member element, [folders]) members: Dict[str, Tuple[Dict[str, Any], List[str]]] = {} @@ -688,9 +948,10 @@ def sync_report_specs(client, *, folders: Optional[Iterable[str]] = None, specs: main_dir, drafts_dir = directory, directory / DRAFTS_DIR main_old, drafts_old = _read_manifest(main_dir), _read_manifest(drafts_dir) - main_new = _new_manifest(main_old, identity, shared) + bound = client_credential_identity(client) + main_new = _new_manifest(main_old, identity, shared, bound) shared_cache = bool(main_new.get("shared")) # §9: a shared cache never holds a zoned spec - drafts_new = _new_manifest(drafts_old, identity, shared) + drafts_new = _new_manifest(drafts_old, identity, shared, bound) drafts_report = SyncReport(cache_dir=str(drafts_dir), folders=folder_qns) if include_drafts else None # Identify every candidate; duplicate labels are skipped (both), with both GUIDs. @@ -785,15 +1046,17 @@ def sync_report_specs(client, *, folders: Optional[Iterable[str]] = None, specs: rep.pruned.append(label) # seen but skipped this time; the skip names the reason continue if old.get("pinned"): - try: - still = _get_full(client, old.get("guid", "")) - except Exception: # noqa: BLE001 - still = None - if still: + state, why = _pinned_element_state(client, old.get("guid", "")) + if state == "found": + tgt_new["entries"][label] = old + continue + if state == "unknown": # a transient fault never drops what the user pinned tgt_new["entries"][label] = old + rep.notes.append(f"{label}: kept; couldn't confirm whether its element {old.get('guid')} " + f"still exists ({why})") continue rep.pruned.append(label) - rep.notes.append(f"{label}: pruned, its element {old.get('guid')} no longer exists") + rep.notes.append(f"{label}: pruned, its element {old.get('guid')} {why}") continue rep.pruned.append(label) rep.notes.append(f"{label}: pruned, no longer a member of any synced folder") @@ -821,10 +1084,12 @@ def sync_report_specs(client, *, folders: Optional[Iterable[str]] = None, specs: return report -def _new_manifest(old: Dict[str, Any], identity: str, shared: Optional[bool]) -> Dict[str, Any]: +def _new_manifest(old: Dict[str, Any], identity: str, shared: Optional[bool], + bound: Optional[str] = None) -> Dict[str, Any]: + """`bound`: the credential identity that synced it; a private manifest loads only for it.""" return {"format": MANIFEST_FORMAT, "shared": bool(old.get("shared")) if shared is None else bool(shared), - "sync_identity": identity, "synced_at": _now(), "entries": {}, "skipped": []} + "sync_identity": identity, "bound_identity": bound, "synced_at": _now(), "entries": {}, "skipped": []} def _resolve_cache_dir(client, override: Optional[str]) -> Path: @@ -1018,12 +1283,65 @@ def read_consumer_index(*, cache_dir: Optional[str] = None, client=None) -> Dict # ── The EGERIA tier: what the registry reads ── -_TIER: Dict[str, Any] = {"key": None, "checked": 0.0, "stamp": None, "specs": {}, "meta": {}, "aliases_dropped": {}} +@dataclass(frozen=True) +class _TierEntry: + """One identity's EGERIA tier, built whole and never changed after (T4): a reload builds a new entry and + swaps it in with one assignment under _TIER_LOCK, so no reader sees one identity's key with another's specs.""" + raw_key: Tuple + key: Any + stamp: Any + checked: float + specs: Any + meta: Dict[str, Any] + skipped: Tuple = () + dir: Optional[str] = None + + +#: Per-identity tier entries, keyed by the full raw key (cache dir and settings, drafts, identity), LRU-bounded. +#: Never one shared slot: a reload for one identity can't hand its half-loaded state, or the previous +#: identity's specs, to a concurrent read for another (the Portal's threaded routes, per-request identities). +TIER_CACHE_SIZE = 64 +_TIERS: "OrderedDict[Tuple, _TierEntry]" = OrderedDict() +_TIER_LOCK = threading.Lock() def _invalidate_tier() -> None: - _TIER["checked"] = 0.0 - _TIER["stamp"] = None + with _TIER_LOCK: + _TIERS.clear() + + +def _tier_raw_key(identity: Optional[str]) -> Tuple: + with_drafts = os.getenv(DRAFTS_ENV, "").strip().lower() in ("1", "true", "yes") + env = _settings_env() + return (os.getenv(CACHE_DIR_ENV), getattr(env, "egeria_view_server_url", None), + getattr(env, "egeria_view_server", None), getattr(env, "pyegeria_user_report_specs_dir", None), + os.getenv("PYEGERIA_USER_REPORT_SPECS_DIR"), os.getenv("PYEGERIA_USER_FORMAT_SETS_DIR"), + with_drafts, identity) + + +def _get_entry(raw_key: Tuple) -> Optional[_TierEntry]: + with _TIER_LOCK: + entry = _TIERS.get(raw_key) + if entry is not None: + _TIERS.move_to_end(raw_key) + return entry + + +def _put_entry(entry: _TierEntry) -> _TierEntry: + with _TIER_LOCK: + _TIERS[entry.raw_key] = entry + _TIERS.move_to_end(entry.raw_key) + while len(_TIERS) > TIER_CACHE_SIZE: + _TIERS.popitem(last=False) + return entry + + +def _peek_tier_entry() -> Optional[_TierEntry]: + """The current identity's entry if one is loaded, without loading (for reads made during a load).""" + try: + return _get_entry(_tier_raw_key(current_spec_identity())) + except Exception: # noqa: BLE001 + return None def _stamp(directory: Path, with_drafts: bool) -> Tuple: @@ -1065,37 +1383,41 @@ def _load_dir(directory: Path, tier_label: str, specs, meta: Dict[str, Any], ski def egeria_tier(*, force: bool = False): - """The EGERIA tier as a FormatSetDict, loaded from the cache manifest(s). Reloaded when a - manifest changes, checked at most once per second. Never raises: a broken cache is an empty + """The EGERIA tier as a FormatSetDict, loaded from the cache manifest(s), for the current spec identity. + Reloaded when a manifest changes, checked at most once per second. Never raises: a broken cache is an empty tier, logged.""" + return _tier_entry(force=force).specs + + +def _tier_entry(*, force: bool = False) -> _TierEntry: + """The current identity's tier entry: its own, never another identity's (T4). The identity is resolved once, + so the entry, its specs and its meta all belong to it.""" from pyegeria.view._output_format_models import FormatSetDict - with_drafts = os.getenv(DRAFTS_ENV, "").strip().lower() in ("1", "true", "yes") - env = _settings_env() - raw_key = (os.getenv(CACHE_DIR_ENV), getattr(env, "egeria_view_server_url", None), - getattr(env, "egeria_view_server", None), getattr(env, "pyegeria_user_report_specs_dir", None), - os.getenv("PYEGERIA_USER_REPORT_SPECS_DIR"), os.getenv("PYEGERIA_USER_FORMAT_SETS_DIR"), - with_drafts) + identity = current_spec_identity() + raw_key = _tier_raw_key(identity) + with_drafts = raw_key[6] now = time.monotonic() - if not force and _TIER.get("raw_key") == raw_key and now - _TIER["checked"] < _RELOAD_INTERVAL_S: - return _TIER["specs"] + entry = _get_entry(raw_key) + if not force and entry is not None and now - entry.checked < _RELOAD_INTERVAL_S: + return entry try: directory = cache_dir() except Exception as exc: # noqa: BLE001 - if _TIER.get("key") != ("error", str(exc)): + if entry is None or entry.key != ("error", str(exc)): logger.warning(f"Report spec cache unavailable: {exc}") - _TIER.update(key=("error", str(exc)), raw_key=raw_key, checked=now, specs=FormatSetDict(), meta={}) - return _TIER["specs"] - key = (str(directory), with_drafts) - _TIER["raw_key"] = raw_key + return _put_entry(_TierEntry(raw_key=raw_key, key=("error", str(exc)), stamp=None, checked=now, + specs=FormatSetDict(), meta={})) + key = (str(directory), with_drafts, identity) stamp = _stamp(directory, with_drafts) - _TIER["checked"] = now - if not force and _TIER.get("key") == key and _TIER.get("stamp") == stamp: - return _TIER["specs"] + if not force and entry is not None and entry.key == key and entry.stamp == stamp: + return _put_entry(replace(entry, checked=now)) specs, meta, skipped = FormatSetDict(), {}, [] if stamp[0] is not None: - shared = bool(_read_manifest(directory).get("shared")) - _load_dir(directory, "main", specs, meta, skipped, shared) - if with_drafts and stamp[1] is not None: + main_manifest = _read_manifest(directory) + if _manifest_loads_for(directory, main_manifest, identity): + _load_dir(directory, "main", specs, meta, skipped, bool(main_manifest.get("shared"))) + if with_drafts and stamp[1] is not None and _manifest_loads_for( + directory / DRAFTS_DIR, _read_manifest(directory / DRAFTS_DIR), identity): dspecs, dmeta = FormatSetDict(), {} _load_dir(directory / DRAFTS_DIR, "drafts", dspecs, dmeta, skipped, False) for label, fs in dspecs.items(): @@ -1106,17 +1428,18 @@ def egeria_tier(*, force: bool = False): meta[label] = dict(dmeta[label], draft=True) for who, why in skipped: logger.warning(f"Report spec cache {directory}: skipping {who}: {why}") - _TIER.update(key=key, stamp=stamp, specs=specs, meta=meta, skipped=skipped, dir=str(directory)) - return specs + return _put_entry(_TierEntry(raw_key=raw_key, key=key, stamp=stamp, checked=now, specs=specs, meta=meta, + skipped=tuple(skipped), dir=str(directory))) def egeria_tier_info(label: str) -> Optional[Dict[str, Any]]: - """Provenance for a label in the EGERIA tier (or a fetched `label@vN`), else None.""" + """Provenance for a label in the EGERIA tier (or a fetched `label@vN`), else None. Read from the caller's own + tier entry, the one its specs came from (T4).""" m = _FETCHED_SUFFIX.match(label or "") - if m and (m.group("label"), int(m.group("version"))) in _FETCHED: - return dict(_FETCHED[(m.group("label"), int(m.group("version")))][1]) - egeria_tier() - info = _TIER.get("meta", {}).get(label) + hit = _fetched_hit(m.group("label"), int(m.group("version"))) if m else None + if hit: + return dict(hit[1]) + info = _tier_entry().meta.get(label) return dict(info) if info else None @@ -1151,17 +1474,50 @@ def spec_footer(info: Optional[Dict[str, Any]]) -> Optional[str]: # ── Fetch on a call (§7a "Fetch on a call") ── -#: Specs fetched with refresh_spec=True, kept for the life of the process, keyed by (label, header -#: version). Addressed internally as "egeria: