diff --git a/.github/workflows/probe-gate.yml b/.github/workflows/probe-gate.yml new file mode 100644 index 00000000..a290a6b6 --- /dev/null +++ b/.github/workflows/probe-gate.yml @@ -0,0 +1,61 @@ +name: Dr.Egeria round-trip probe gate + +# Before a pyegeria release: a scratch Egeria (in this runner only, never a shared platform) and a +# round trip of every Dr.Egeria Create command (PostgreSQL repository on a private network, Core Content +# Pack, the pyegeria-side REJECTED attributes sent), compared with the committed baseline +# (scripts/dr_egeria_roundtrip_probe/baseline/.json). See that folder's README. + +on: + workflow_dispatch: + inputs: + digest: + description: "odpi/egeria-platform image digest (default: the pin in scratch.py)" + required: false + default: "" + pull_request: + types: [opened, synchronize, reopened, labeled] + +permissions: + contents: read + +jobs: + gate: + if: >- + github.event_name == 'workflow_dispatch' || + contains(github.event.pull_request.labels.*.name, 'release') || + startsWith(github.base_ref, 'release/') + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Install uv + uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0 + with: + enable-cache: true + version: "latest" + + - name: Set up Python 3.13 + uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: "3.13" + + - name: Install dependencies + run: uv sync --all-extras --dev + + - name: Probe gate (scratch up, round trip, compare, scratch down) + env: + DIGEST: ${{ github.event.inputs.digest }} + run: | + args=(--out "$RUNNER_TEMP/roundtrip-probe") + if [ -n "$DIGEST" ]; then args+=(--digest "$DIGEST"); fi + uv run python scripts/dr_egeria_roundtrip_probe/gate.py "${args[@]}" + + - name: Upload results + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: roundtrip-probe-results + path: | + ${{ runner.temp }}/roundtrip-probe/RESULTS-* + if-no-files-found: warn diff --git a/docs/RELEASING.md b/docs/RELEASING.md new file mode 100644 index 00000000..42ca9bc0 --- /dev/null +++ b/docs/RELEASING.md @@ -0,0 +1,21 @@ +# Releasing pyegeria + +The release checklist, in order. The release itself (PyPI, GitHub release) is run by +`.github/workflows/release.yml` when a `v` tag is pushed. + +1. **Feature PRs merged** to `main` (squash-merge). +2. **Version bump PR**: `pyproject.toml` and `uv.lock` only (`uv lock`), CI green, squash-merged. +3. **3.13 micro-tests** pass on the bump commit (`uv run -p 3.13 pytest tests/micro-tests/`). +4. **Dr.Egeria round-trip probe gate green, or its failure explained in the PR.** Run the + "Dr.Egeria round-trip probe gate" workflow (`workflow_dispatch`, or label the PR `release`), or locally + `uv run -p 3.13 python scripts/dr_egeria_roundtrip_probe/gate.py` on a host with Docker (attach + `RESULTS-.md` and the gate output to the PR). Improvements mean the PR updates + `scripts/dr_egeria_roundtrip_probe/baseline/.json` (`--rebaseline`, one-line reason per changed + row). A new Egeria digest means a new baseline file. See `scripts/dr_egeria_roundtrip_probe/README.md`; + the probe only ever runs against its own scratch Egeria. +5. **Check PyPI first:** `https://pypi.org/pypi/pyegeria//json` must return 404 (a local upload + would make the tag-triggered publish fail). +6. **Tag** the merge commit with a lightweight `v` tag and push it to `upstream`; approve the + `release` environment deployment when asked. +7. **Verify** the version on PyPI and the GitHub release; prepend any behaviour change hidden in a PR + title to the release notes. diff --git a/md_processing/data/compact_commands/commands_metadataexpert_compact.json b/md_processing/data/compact_commands/commands_metadataexpert_compact.json new file mode 100644 index 00000000..4ff98d67 --- /dev/null +++ b/md_processing/data/compact_commands/commands_metadataexpert_compact.json @@ -0,0 +1,1215 @@ +{ + "family": "MetadataExpert", + "exported": "", + "attribute_definitions": { + "Type Def Name": { + "variable_name": "type_def_name", + "inUpdate": true, + "attr_labels": "", + "examples": "Recipe", + "default_value": "", + "valid_values": [], + "existing_element": "", + "description": "Name of the open metadata type definition (entity, relationship or classification).", + "generated": false, + "style": "Simple", + "user_specified": true, + "unique": false, + "input_required": true, + "min_cardinality": 1, + "max_cardinality": 1, + "level": "Advanced", + "Journal Entry": "Added 2026-11-12 -- MetadataExpert family." + }, + "Type Def GUID": { + "variable_name": "type_def_guid", + "inUpdate": true, + "attr_labels": "", + "examples": "", + "default_value": "", + "valid_values": [], + "existing_element": "", + "description": "GUID of the type definition. If omitted, it is looked up from the Type Def Name.", + "generated": false, + "style": "GUID", + "user_specified": true, + "unique": false, + "input_required": false, + "min_cardinality": 0, + "max_cardinality": 1, + "level": "Advanced", + "Journal Entry": "Added 2026-11-12 -- MetadataExpert family." + }, + "Type Def Category": { + "variable_name": "type_def_category", + "inUpdate": true, + "attr_labels": "", + "examples": "", + "default_value": "Entity", + "valid_values": [ + "Entity", + "Relationship", + "Classification" + ], + "existing_element": "", + "description": "Kind of type definition to create.", + "generated": false, + "style": "Valid Value", + "user_specified": true, + "unique": false, + "input_required": false, + "min_cardinality": 0, + "max_cardinality": 1, + "level": "Advanced", + "Journal Entry": "Added 2026-11-12 -- MetadataExpert family." + }, + "Super Type": { + "variable_name": "super_type", + "inUpdate": true, + "attr_labels": "", + "examples": "Referenceable", + "default_value": "", + "valid_values": [], + "existing_element": "", + "description": "Name of the supertype (entity and classification definitions).", + "generated": false, + "style": "Simple", + "user_specified": true, + "unique": false, + "input_required": false, + "min_cardinality": 0, + "max_cardinality": 1, + "level": "Advanced", + "Journal Entry": "Added 2026-11-12 -- MetadataExpert family." + }, + "Type Def Description": { + "variable_name": "type_def_description", + "inUpdate": true, + "attr_labels": "", + "examples": "", + "default_value": "", + "valid_values": [], + "existing_element": "", + "description": "Description of the type definition.", + "generated": false, + "style": "Simple", + "user_specified": true, + "unique": false, + "input_required": false, + "min_cardinality": 0, + "max_cardinality": 1, + "level": "Advanced", + "Journal Entry": "Added 2026-11-12 -- MetadataExpert family." + }, + "Type Def Attributes": { + "variable_name": "type_def_attributes", + "inUpdate": true, + "attr_labels": "", + "examples": "servings: int", + "default_value": "", + "valid_values": [], + "existing_element": "", + "description": "One attribute per line as 'attributeName: typeName'. typeName is a primitive type (string, int, long, boolean, date, float, double, short, byte, char, bigint, bigdecimal) or the name of an existing enum definition.", + "generated": false, + "style": "Dictionary", + "user_specified": true, + "unique": false, + "input_required": false, + "min_cardinality": 0, + "max_cardinality": 1, + "level": "Advanced", + "Journal Entry": "Added 2026-11-12 -- MetadataExpert family." + }, + "Valid Entity Types": { + "variable_name": "valid_entity_types", + "inUpdate": true, + "attr_labels": "", + "examples": "", + "default_value": "", + "valid_values": [], + "existing_element": "", + "description": "Classification definitions only: the entity types the classification can be attached to (one per line or comma separated).", + "generated": false, + "style": "Simple", + "user_specified": true, + "unique": false, + "input_required": false, + "min_cardinality": 0, + "max_cardinality": 1, + "level": "Advanced", + "Journal Entry": "Added 2026-11-12 -- MetadataExpert family." + }, + "Propagatable": { + "variable_name": "propagatable", + "inUpdate": true, + "attr_labels": "", + "examples": "", + "default_value": "", + "valid_values": [], + "existing_element": "", + "description": "Classification definitions only: whether the classification propagates.", + "generated": false, + "style": "Bool", + "user_specified": true, + "unique": false, + "input_required": false, + "min_cardinality": 0, + "max_cardinality": 1, + "level": "Advanced", + "Journal Entry": "Added 2026-11-12 -- MetadataExpert family." + }, + "End1 Entity Type": { + "variable_name": "end1_entity_type", + "inUpdate": true, + "attr_labels": "", + "examples": "", + "default_value": "", + "valid_values": [], + "existing_element": "", + "description": "Relationship definitions only: entity type at end 1.", + "generated": false, + "style": "Simple", + "user_specified": true, + "unique": false, + "input_required": false, + "min_cardinality": 0, + "max_cardinality": 1, + "level": "Advanced", + "Journal Entry": "Added 2026-11-12 -- MetadataExpert family." + }, + "End1 Attribute Name": { + "variable_name": "end1_attribute_name", + "inUpdate": true, + "attr_labels": "", + "examples": "", + "default_value": "", + "valid_values": [], + "existing_element": "", + "description": "Relationship definitions only: name the relationship has at end 1.", + "generated": false, + "style": "Simple", + "user_specified": true, + "unique": false, + "input_required": false, + "min_cardinality": 0, + "max_cardinality": 1, + "level": "Advanced", + "Journal Entry": "Added 2026-11-12 -- MetadataExpert family." + }, + "End2 Entity Type": { + "variable_name": "end2_entity_type", + "inUpdate": true, + "attr_labels": "", + "examples": "", + "default_value": "", + "valid_values": [], + "existing_element": "", + "description": "Relationship definitions only: entity type at end 2.", + "generated": false, + "style": "Simple", + "user_specified": true, + "unique": false, + "input_required": false, + "min_cardinality": 0, + "max_cardinality": 1, + "level": "Advanced", + "Journal Entry": "Added 2026-11-12 -- MetadataExpert family." + }, + "End2 Attribute Name": { + "variable_name": "end2_attribute_name", + "inUpdate": true, + "attr_labels": "", + "examples": "", + "default_value": "", + "valid_values": [], + "existing_element": "", + "description": "Relationship definitions only: name the relationship has at end 2.", + "generated": false, + "style": "Simple", + "user_specified": true, + "unique": false, + "input_required": false, + "min_cardinality": 0, + "max_cardinality": 1, + "level": "Advanced", + "Journal Entry": "Added 2026-11-12 -- MetadataExpert family." + }, + "Apply To Version": { + "variable_name": "apply_to_version", + "inUpdate": true, + "attr_labels": "", + "examples": "1", + "default_value": "", + "valid_values": [], + "existing_element": "", + "description": "The version of the type definition this update applies to.", + "generated": false, + "style": "Simple", + "user_specified": true, + "unique": false, + "input_required": true, + "min_cardinality": 1, + "max_cardinality": 1, + "level": "Advanced", + "Journal Entry": "Added 2026-11-12 -- MetadataExpert family." + }, + "Enum Def Name": { + "variable_name": "enum_def_name", + "inUpdate": true, + "attr_labels": "", + "examples": "CuisineType", + "default_value": "", + "valid_values": [], + "existing_element": "", + "description": "Name of the enum definition.", + "generated": false, + "style": "Simple", + "user_specified": true, + "unique": false, + "input_required": true, + "min_cardinality": 1, + "max_cardinality": 1, + "level": "Advanced", + "Journal Entry": "Added 2026-11-12 -- MetadataExpert family." + }, + "Enum Def GUID": { + "variable_name": "enum_def_guid", + "inUpdate": true, + "attr_labels": "", + "examples": "", + "default_value": "", + "valid_values": [], + "existing_element": "", + "description": "GUID of the enum definition. If omitted, it is looked up from the Enum Def Name.", + "generated": false, + "style": "GUID", + "user_specified": true, + "unique": false, + "input_required": false, + "min_cardinality": 0, + "max_cardinality": 1, + "level": "Advanced", + "Journal Entry": "Added 2026-11-12 -- MetadataExpert family." + }, + "Enum Def Description": { + "variable_name": "enum_def_description", + "inUpdate": true, + "attr_labels": "", + "examples": "", + "default_value": "", + "valid_values": [], + "existing_element": "", + "description": "Description of the enum definition.", + "generated": false, + "style": "Simple", + "user_specified": true, + "unique": false, + "input_required": false, + "min_cardinality": 0, + "max_cardinality": 1, + "level": "Advanced", + "Journal Entry": "Added 2026-11-12 -- MetadataExpert family." + }, + "Enum Values": { + "variable_name": "enum_values", + "inUpdate": true, + "attr_labels": "", + "examples": "Unclassified: The cuisine has not been recorded.", + "default_value": "", + "valid_values": [], + "existing_element": "", + "description": "One enum value per line as 'value: description'. Ordinals are assigned 0, 1, 2... in line order.", + "generated": false, + "style": "Dictionary", + "user_specified": true, + "unique": false, + "input_required": true, + "min_cardinality": 1, + "max_cardinality": 1, + "level": "Advanced", + "Journal Entry": "Added 2026-11-12 -- MetadataExpert family." + }, + "Enum Default Value": { + "variable_name": "enum_default_value", + "inUpdate": true, + "attr_labels": "", + "examples": "", + "default_value": "", + "valid_values": [], + "existing_element": "", + "description": "Which of the Enum Values is the default (defaults to the first).", + "generated": false, + "style": "Simple", + "user_specified": true, + "unique": false, + "input_required": false, + "min_cardinality": 0, + "max_cardinality": 1, + "level": "Advanced", + "Journal Entry": "Added 2026-11-12 -- MetadataExpert family." + }, + "Count Element Type Name": { + "variable_name": "count_element_type_name", + "inUpdate": true, + "attr_labels": "", + "examples": "GlossaryTerm", + "default_value": "", + "valid_values": [], + "existing_element": "", + "description": "Only count elements of this metadata element type (and subtypes).", + "generated": false, + "style": "Simple", + "user_specified": true, + "unique": false, + "input_required": false, + "min_cardinality": 0, + "max_cardinality": 1, + "level": "Advanced", + "Journal Entry": "Added 2026-11-12 -- MetadataExpert family." + }, + "Count Search String": { + "variable_name": "count_search_string", + "inUpdate": true, + "attr_labels": "", + "examples": "", + "default_value": "", + "valid_values": [], + "existing_element": "", + "description": "Only count elements matching this search string.", + "generated": false, + "style": "Simple", + "user_specified": true, + "unique": false, + "input_required": false, + "min_cardinality": 0, + "max_cardinality": 1, + "level": "Advanced", + "Journal Entry": "Added 2026-11-12 -- MetadataExpert family." + }, + "Count Push Down": { + "variable_name": "count_push_down", + "inUpdate": true, + "attr_labels": "", + "examples": "", + "default_value": "", + "valid_values": [], + "existing_element": "", + "description": "True: the repository counts rows itself (fast, skips per-element visibility checks). False: the server retrieves and counts matching elements. Omit for the server default.", + "generated": false, + "style": "Bool", + "user_specified": true, + "unique": false, + "input_required": false, + "min_cardinality": 0, + "max_cardinality": 1, + "level": "Advanced", + "Journal Entry": "Added 2026-11-12 -- MetadataExpert family." + }, + "Metadata Element GUID": { + "variable_name": "metadata_element_guid", + "inUpdate": true, + "attr_labels": "", + "examples": "", + "default_value": "", + "valid_values": [], + "existing_element": "", + "description": "GUID of the metadata element.", + "generated": false, + "style": "GUID", + "user_specified": true, + "unique": false, + "input_required": true, + "min_cardinality": 1, + "max_cardinality": 1, + "level": "Advanced", + "Journal Entry": "Added 2026-11-12 -- MetadataExpert family." + }, + "Instance Type Name": { + "variable_name": "instance_type_name", + "inUpdate": true, + "attr_labels": "", + "examples": "GlossaryTerm", + "default_value": "", + "valid_values": [], + "existing_element": "", + "description": "Open metadata type name of the instance (element) to create.", + "generated": false, + "style": "Simple", + "user_specified": true, + "unique": false, + "input_required": true, + "min_cardinality": 1, + "max_cardinality": 1, + "level": "Advanced", + "Journal Entry": "Added 2026-11-12 -- MetadataExpert family (instance commands)." + }, + "Instance Properties": { + "variable_name": "instance_properties", + "inUpdate": true, + "attr_labels": "", + "examples": "qualifiedName: Term::example", + "default_value": "", + "valid_values": [], + "existing_element": "", + "description": "One property per line as 'name: value'. true/false become boolean, whole numbers become int, anything else is a string.", + "generated": false, + "style": "Dictionary", + "user_specified": true, + "unique": false, + "input_required": false, + "min_cardinality": 0, + "max_cardinality": 1, + "level": "Advanced", + "Journal Entry": "Added 2026-11-12 -- MetadataExpert family (instance commands)." + }, + "Replace Properties": { + "variable_name": "replace_properties", + "inUpdate": true, + "attr_labels": "", + "examples": "", + "default_value": "", + "valid_values": [], + "existing_element": "", + "description": "True: the supplied properties replace all existing ones. False (default): they are merged.", + "generated": false, + "style": "Bool", + "user_specified": true, + "unique": false, + "input_required": false, + "min_cardinality": 0, + "max_cardinality": 1, + "level": "Advanced", + "Journal Entry": "Added 2026-11-12 -- MetadataExpert family (instance commands)." + }, + "Anchor GUID": { + "variable_name": "anchor_guid", + "inUpdate": true, + "attr_labels": "", + "examples": "", + "default_value": "", + "valid_values": [], + "existing_element": "", + "description": "GUID of the element that anchors the new instance.", + "generated": false, + "style": "Simple", + "user_specified": true, + "unique": false, + "input_required": false, + "min_cardinality": 0, + "max_cardinality": 1, + "level": "Advanced", + "Journal Entry": "Added 2026-11-12 -- MetadataExpert family (instance commands)." + }, + "Is Own Anchor": { + "variable_name": "is_own_anchor", + "inUpdate": true, + "attr_labels": "", + "examples": "", + "default_value": "true", + "valid_values": [], + "existing_element": "", + "description": "A flag indicating if the element is its own anchor or is anchored to a different element.", + "generated": false, + "style": "Bool", + "user_specified": true, + "unique": false, + "input_required": false, + "min_cardinality": 1, + "max_cardinality": 1, + "level": "Advanced", + "Journal Entry": "" + }, + "Parent GUID": { + "variable_name": "parent_guid", + "inUpdate": true, + "attr_labels": "", + "examples": "", + "default_value": "", + "valid_values": [], + "existing_element": "", + "description": "GUID of a parent element to link the new instance to.", + "generated": false, + "style": "Simple", + "user_specified": true, + "unique": false, + "input_required": false, + "min_cardinality": 0, + "max_cardinality": 1, + "level": "Advanced", + "Journal Entry": "Added 2026-11-12 -- MetadataExpert family (instance commands)." + }, + "Parent Relationship Type Name": { + "variable_name": "parent_relationship_type_name", + "inUpdate": true, + "attr_labels": "", + "examples": "", + "default_value": "", + "valid_values": [], + "existing_element": "", + "description": "The type of parent relationship.", + "generated": false, + "style": "Simple", + "user_specified": true, + "unique": false, + "input_required": false, + "min_cardinality": 0, + "max_cardinality": 1, + "level": "Advanced", + "Journal Entry": "" + }, + "Delete Method": { + "variable_name": "delete_method", + "inUpdate": true, + "attr_labels": "", + "examples": "", + "default_value": "SOFT_DELETE", + "valid_values": [ + "SOFT_DELETE", + "PURGE", + "ARCHIVE", + "LOOK_FOR_LINEAGE" + ], + "existing_element": "", + "description": "How to delete.", + "generated": false, + "style": "Valid Value", + "user_specified": true, + "unique": false, + "input_required": false, + "min_cardinality": 0, + "max_cardinality": 1, + "level": "Advanced", + "Journal Entry": "Added 2026-11-12 -- MetadataExpert family (instance commands)." + }, + "Relationship Type Name": { + "variable_name": "relationship_type_name", + "inUpdate": true, + "attr_labels": "", + "examples": "SemanticAssignment", + "default_value": "", + "valid_values": [], + "existing_element": "", + "description": "Open metadata type name of the relationship to create.", + "generated": false, + "style": "Simple", + "user_specified": true, + "unique": false, + "input_required": true, + "min_cardinality": 1, + "max_cardinality": 1, + "level": "Advanced", + "Journal Entry": "Added 2026-11-12 -- MetadataExpert family (instance commands)." + }, + "Element 1 GUID": { + "variable_name": "element_1_guid", + "inUpdate": true, + "attr_labels": "", + "examples": "", + "default_value": "", + "valid_values": [], + "existing_element": "", + "description": "GUID of the element at end 1 of the relationship.", + "generated": false, + "style": "GUID", + "user_specified": true, + "unique": false, + "input_required": true, + "min_cardinality": 1, + "max_cardinality": 1, + "level": "Advanced", + "Journal Entry": "Added 2026-11-12 -- MetadataExpert family (instance commands)." + }, + "Element 2 GUID": { + "variable_name": "element_2_guid", + "inUpdate": true, + "attr_labels": "", + "examples": "", + "default_value": "", + "valid_values": [], + "existing_element": "", + "description": "GUID of the element at end 2 of the relationship.", + "generated": false, + "style": "GUID", + "user_specified": true, + "unique": false, + "input_required": true, + "min_cardinality": 1, + "max_cardinality": 1, + "level": "Advanced", + "Journal Entry": "Added 2026-11-12 -- MetadataExpert family (instance commands)." + }, + "Relationship Properties": { + "variable_name": "relationship_properties", + "inUpdate": true, + "attr_labels": "", + "examples": "", + "default_value": "", + "valid_values": [], + "existing_element": "", + "description": "One property per line as 'name: value' (same typing rules as Instance Properties).", + "generated": false, + "style": "Dictionary", + "user_specified": true, + "unique": false, + "input_required": false, + "min_cardinality": 0, + "max_cardinality": 1, + "level": "Advanced", + "Journal Entry": "Added 2026-11-12 -- MetadataExpert family (instance commands)." + }, + "Relationship GUID": { + "variable_name": "relationship_guid", + "inUpdate": true, + "attr_labels": "", + "examples": "", + "default_value": "", + "valid_values": [], + "existing_element": "", + "description": "GUID of the relationship.", + "generated": false, + "style": "GUID", + "user_specified": true, + "unique": false, + "input_required": true, + "min_cardinality": 1, + "max_cardinality": 1, + "level": "Advanced", + "Journal Entry": "Added 2026-11-12 -- MetadataExpert family (instance commands)." + }, + "Classification Name": { + "variable_name": "classification_name", + "inUpdate": true, + "attr_labels": "", + "examples": "Confidentiality", + "default_value": "", + "valid_values": [], + "existing_element": "", + "description": "Open metadata type name of the classification.", + "generated": false, + "style": "Simple", + "user_specified": true, + "unique": false, + "input_required": true, + "min_cardinality": 1, + "max_cardinality": 1, + "level": "Advanced", + "Journal Entry": "Added 2026-11-12 -- MetadataExpert family (instance commands)." + }, + "Classification Properties": { + "variable_name": "classification_properties", + "inUpdate": true, + "attr_labels": "", + "examples": "", + "default_value": "", + "valid_values": [], + "existing_element": "", + "description": "One property per line as 'name: value' (same typing rules as Instance Properties).", + "generated": false, + "style": "Dictionary", + "user_specified": true, + "unique": false, + "input_required": false, + "min_cardinality": 0, + "max_cardinality": 1, + "level": "Advanced", + "Journal Entry": "Added 2026-11-12 -- MetadataExpert family (instance commands)." + }, + "Parent at End1": { + "variable_name": "parent_at_end1", + "inUpdate": true, + "attr_labels": "", + "examples": "", + "default_value": "true", + "valid_values": [], + "existing_element": "", + "description": "A flag indicating if the parent is at end1 of the relationship", + "generated": false, + "style": "Bool", + "user_specified": true, + "unique": false, + "input_required": false, + "min_cardinality": 1, + "max_cardinality": 1, + "level": "Advanced", + "Journal Entry": "" + }, + "Confirm Delete GUID": { + "variable_name": "confirm_delete_guid", + "inUpdate": true, + "attr_labels": "", + "examples": "", + "default_value": "", + "valid_values": [], + "existing_element": "", + "description": "Required when Delete Method is PURGE or HARD_DELETE: repeat the GUID being deleted. A purge cannot be undone and must have been soft deleted first.", + "generated": false, + "style": "GUID", + "user_specified": true, + "unique": false, + "input_required": false, + "min_cardinality": 1, + "max_cardinality": 1, + "level": "Advanced", + "Journal Entry": "Added 2026-11-13 -- Architect read-back fix 2.", + "display_name": "Confirm Delete GUID" + }, + "Confirm Type Def Name": { + "variable_name": "confirm_type_def_name", + "inUpdate": true, + "attr_labels": "", + "examples": "", + "default_value": "", + "valid_values": [], + "existing_element": "", + "description": "Required for Update Type Def and Delete Type Def: repeat Type Def Name exactly (case-sensitive). Changing a type definition affects every user and app on the platform.", + "generated": false, + "style": "Simple", + "user_specified": true, + "unique": false, + "input_required": false, + "min_cardinality": 1, + "max_cardinality": 1, + "level": "Advanced", + "Journal Entry": "Added 2026-11-13 -- Architect typedef guard.", + "display_name": "Confirm Delete GUID" + }, + "Confirm Enum Def Name": { + "variable_name": "confirm_enum_def_name", + "inUpdate": true, + "attr_labels": "", + "examples": "", + "default_value": "", + "valid_values": [], + "existing_element": "", + "description": "Required for Delete Enum Def: repeat Enum Def Name exactly (case-sensitive). Changing an enum definition affects every user and app on the platform.", + "generated": false, + "style": "Simple", + "user_specified": true, + "unique": false, + "input_required": false, + "min_cardinality": 1, + "max_cardinality": 1, + "level": "Advanced", + "Journal Entry": "Added 2026-11-13 -- Architect typedef guard.", + "display_name": "Confirm Delete GUID" + } + }, + "bundles": {}, + "commands": { + "Create Type Def": { + "display_name": "type_def", + "qn_prefix": "", + "alternate_names": [], + "family": "MetadataExpert", + "description": "Add a new open metadata type definition (entity, relationship or classification) to the type system. Not an upsert: types cannot be removed once instances exist. Creates a new item each time it runs (not idempotent): never put it in a re-run or heal batch.", + "verb": "Create", + "upsert": false, + "attach": false, + "level": "Advanced", + "find_method": "", + "find_constraints": "", + "extra_find": "", + "extra_constraints": "", + "OM_TYPE": "", + "bundle": "", + "custom_attributes": [ + "Type Def Name", + "Type Def Category", + "Super Type", + "Type Def Description", + "Type Def Attributes", + "Valid Entity Types", + "Propagatable", + "End1 Entity Type", + "End1 Attribute Name", + "End2 Entity Type", + "End2 Attribute Name" + ], + "Journal Entry": "Added 2026-11-12 -- MetadataExpert family. ", + "idempotent": false, + "chat_allowed": false + }, + "Update Type Def": { + "display_name": "type_def", + "qn_prefix": "", + "alternate_names": [], + "family": "MetadataExpert", + "description": "Patch a type definition that was added through the API. WARNING: Egeria currently rejects every patch (OMRS-REPOSITORY-400-069, ISSUE-124). PLATFORM-WIDE: this changes the type system for every user and app, so Confirm Type Def Name must repeat the name exactly or nothing is changed.", + "verb": "Update", + "upsert": false, + "attach": false, + "level": "Advanced", + "find_method": "", + "find_constraints": "", + "extra_find": "", + "extra_constraints": "", + "OM_TYPE": "", + "bundle": "", + "custom_attributes": [ + "Type Def Name", + "Type Def GUID", + "Apply To Version", + "Type Def Description", + "Type Def Attributes", + "Confirm Type Def Name" + ], + "Journal Entry": "Added 2026-11-12 -- MetadataExpert family. ", + "idempotent": false, + "chat_allowed": false + }, + "Delete Type Def": { + "display_name": "type_def", + "qn_prefix": "", + "alternate_names": [], + "family": "MetadataExpert", + "description": "Delete a type definition that was added through the API. Fails while instances of the type exist or another type refers to it. PLATFORM-WIDE: this changes the type system for every user and app, so Confirm Type Def Name must repeat the name exactly or nothing is changed.", + "verb": "Delete", + "upsert": false, + "attach": false, + "level": "Advanced", + "find_method": "", + "find_constraints": "", + "extra_find": "", + "extra_constraints": "", + "OM_TYPE": "", + "bundle": "", + "custom_attributes": [ + "Type Def Name", + "Type Def GUID", + "Confirm Type Def Name" + ], + "Journal Entry": "Added 2026-11-12 -- MetadataExpert family. ", + "idempotent": false, + "chat_allowed": false + }, + "Create Enum Def": { + "display_name": "enum_def", + "qn_prefix": "", + "alternate_names": [], + "family": "MetadataExpert", + "description": "Add a new enum definition, usable as an attribute type in type definitions. WARNING: an enum added through the API cannot be removed again (ISSUE-125). Creates a new item each time it runs (not idempotent): never put it in a re-run or heal batch.", + "verb": "Create", + "upsert": false, + "attach": false, + "level": "Advanced", + "find_method": "", + "find_constraints": "", + "extra_find": "", + "extra_constraints": "", + "OM_TYPE": "", + "bundle": "", + "custom_attributes": [ + "Enum Def Name", + "Enum Def Description", + "Enum Values", + "Enum Default Value" + ], + "Journal Entry": "Added 2026-11-12 -- MetadataExpert family. ", + "idempotent": false, + "chat_allowed": false + }, + "Delete Enum Def": { + "display_name": "enum_def", + "qn_prefix": "", + "alternate_names": [], + "family": "MetadataExpert", + "description": "Delete an enum definition. WARNING: currently fails on Egeria with a 500 'unknown TypeDef' (ISSUE-125). PLATFORM-WIDE: this changes the type system for every user and app, so Confirm Enum Def Name must repeat the name exactly or nothing is changed.", + "verb": "Delete", + "upsert": false, + "attach": false, + "level": "Advanced", + "find_method": "", + "find_constraints": "", + "extra_find": "", + "extra_constraints": "", + "OM_TYPE": "", + "bundle": "", + "custom_attributes": [ + "Enum Def Name", + "Enum Def GUID", + "Confirm Enum Def Name" + ], + "Journal Entry": "Added 2026-11-12 -- MetadataExpert family. ", + "idempotent": false, + "chat_allowed": false + }, + "Count Metadata Elements": { + "display_name": "metadata_elements", + "qn_prefix": "", + "alternate_names": [], + "family": "MetadataExpert", + "description": "Return the number of metadata elements matching the criteria, without retrieving them.", + "verb": "Count", + "upsert": false, + "attach": false, + "level": "Advanced", + "find_method": "", + "find_constraints": "", + "extra_find": "", + "extra_constraints": "", + "OM_TYPE": "", + "bundle": "", + "custom_attributes": [ + "Count Element Type Name", + "Count Search String", + "Count Push Down" + ], + "Journal Entry": "Added 2026-11-12 -- MetadataExpert family. ", + "idempotent": false, + "chat_allowed": false + }, + "Publish Metadata Element": { + "display_name": "metadata_element", + "qn_prefix": "", + "alternate_names": [], + "family": "MetadataExpert", + "description": "Increase the visibility of a metadata element by updating its zone membership.", + "verb": "Publish", + "upsert": false, + "attach": false, + "level": "Advanced", + "find_method": "", + "find_constraints": "", + "extra_find": "", + "extra_constraints": "", + "OM_TYPE": "", + "bundle": "", + "custom_attributes": [ + "Metadata Element GUID" + ], + "Journal Entry": "Added 2026-11-12 -- MetadataExpert family. ", + "idempotent": false, + "chat_allowed": false + }, + "Update Instance": { + "display_name": "instance", + "qn_prefix": "", + "alternate_names": [], + "family": "MetadataExpert", + "description": "Update the properties of a metadata element (instance).", + "verb": "Update", + "upsert": false, + "attach": false, + "level": "Advanced", + "find_method": "", + "find_constraints": "", + "extra_find": "", + "extra_constraints": "", + "OM_TYPE": "", + "bundle": "", + "custom_attributes": [ + "Metadata Element GUID", + "Instance Properties", + "Replace Properties" + ], + "Journal Entry": "Added 2026-11-12 -- MetadataExpert family (instance commands).", + "idempotent": false, + "chat_allowed": false + }, + "Delete Instance": { + "display_name": "instance", + "qn_prefix": "", + "alternate_names": [], + "family": "MetadataExpert", + "description": "Delete a metadata element (instance). Defaults to SOFT_DELETE. PURGE is permanent and only works on an element already soft deleted; it also requires Confirm Delete GUID to repeat the GUID.", + "verb": "Delete", + "upsert": false, + "attach": false, + "level": "Advanced", + "find_method": "", + "find_constraints": "", + "extra_find": "", + "extra_constraints": "", + "OM_TYPE": "", + "bundle": "", + "custom_attributes": [ + "Metadata Element GUID", + "Delete Method", + "Confirm Delete GUID" + ], + "Journal Entry": "Added 2026-11-12 -- MetadataExpert family (instance commands).", + "idempotent": false, + "chat_allowed": false + }, + "Link Instances": { + "display_name": "instances", + "qn_prefix": "", + "alternate_names": [], + "family": "MetadataExpert", + "description": "Create a relationship between two metadata elements. Not deduplicated: running it twice may create two relationships. Creates a new item each time it runs (not idempotent): never put it in a re-run or heal batch.", + "verb": "Link", + "upsert": false, + "attach": false, + "level": "Advanced", + "find_method": "", + "find_constraints": "", + "extra_find": "", + "extra_constraints": "", + "OM_TYPE": "", + "bundle": "", + "custom_attributes": [ + "Relationship Type Name", + "Element 1 GUID", + "Element 2 GUID", + "Relationship Properties" + ], + "Journal Entry": "Added 2026-11-12 -- MetadataExpert family (instance commands).", + "idempotent": false, + "chat_allowed": false + }, + "Unlink Instances": { + "display_name": "instances", + "qn_prefix": "", + "alternate_names": [], + "family": "MetadataExpert", + "description": "Delete a relationship between two metadata elements. Defaults to SOFT_DELETE. PURGE is permanent and only works on a relationship already soft deleted; it also requires Confirm Delete GUID to repeat the Relationship GUID.", + "verb": "Unlink", + "upsert": false, + "attach": false, + "level": "Advanced", + "find_method": "", + "find_constraints": "", + "extra_find": "", + "extra_constraints": "", + "OM_TYPE": "", + "bundle": "", + "custom_attributes": [ + "Relationship GUID", + "Delete Method", + "Confirm Delete GUID" + ], + "Journal Entry": "Added 2026-11-12 -- MetadataExpert family (instance commands).", + "idempotent": false, + "chat_allowed": false + }, + "Update Relationship": { + "display_name": "relationship", + "qn_prefix": "", + "alternate_names": [], + "family": "MetadataExpert", + "description": "Update the properties of a relationship.", + "verb": "Update", + "upsert": false, + "attach": false, + "level": "Advanced", + "find_method": "", + "find_constraints": "", + "extra_find": "", + "extra_constraints": "", + "OM_TYPE": "", + "bundle": "", + "custom_attributes": [ + "Relationship GUID", + "Relationship Properties", + "Replace Properties" + ], + "Journal Entry": "Added 2026-11-12 -- MetadataExpert family (instance commands).", + "idempotent": false, + "chat_allowed": false + }, + "Classify Instance": { + "display_name": "instance", + "qn_prefix": "", + "alternate_names": [], + "family": "MetadataExpert", + "description": "Add a classification to a metadata element.", + "verb": "Classify", + "upsert": false, + "attach": false, + "level": "Advanced", + "find_method": "", + "find_constraints": "", + "extra_find": "", + "extra_constraints": "", + "OM_TYPE": "", + "bundle": "", + "custom_attributes": [ + "Metadata Element GUID", + "Classification Name", + "Classification Properties" + ], + "Journal Entry": "Added 2026-11-12 -- MetadataExpert family (instance commands).", + "idempotent": false, + "chat_allowed": false + }, + "Reclassify Instance": { + "display_name": "instance", + "qn_prefix": "", + "alternate_names": [], + "family": "MetadataExpert", + "description": "Change the properties of a classification already on a metadata element.", + "verb": "Reclassify", + "upsert": false, + "attach": false, + "level": "Advanced", + "find_method": "", + "find_constraints": "", + "extra_find": "", + "extra_constraints": "", + "OM_TYPE": "", + "bundle": "", + "custom_attributes": [ + "Metadata Element GUID", + "Classification Name", + "Classification Properties", + "Replace Properties" + ], + "Journal Entry": "Added 2026-11-12 -- MetadataExpert family (instance commands).", + "idempotent": false, + "chat_allowed": false + }, + "Declassify Instance": { + "display_name": "instance", + "qn_prefix": "", + "alternate_names": [], + "family": "MetadataExpert", + "description": "Remove a classification from a metadata element.", + "verb": "Declassify", + "upsert": false, + "attach": false, + "level": "Advanced", + "find_method": "", + "find_constraints": "", + "extra_find": "", + "extra_constraints": "", + "OM_TYPE": "", + "bundle": "", + "custom_attributes": [ + "Metadata Element GUID", + "Classification Name" + ], + "Journal Entry": "Added 2026-11-12 -- MetadataExpert family (instance commands).", + "idempotent": false, + "chat_allowed": false + }, + "Create Instance": { + "display_name": "instance", + "qn_prefix": "", + "alternate_names": [], + "family": "MetadataExpert", + "description": "Create a metadata element (instance) of any open metadata type through the raw metadata-expert API. Not an upsert: every run creates a new element. Creates a new item each time it runs (not idempotent): never put it in a re-run or heal batch.", + "verb": "Create", + "upsert": false, + "attach": false, + "level": "Advanced", + "find_method": "", + "find_constraints": "", + "extra_find": "", + "extra_constraints": "", + "OM_TYPE": "", + "bundle": "", + "custom_attributes": [ + "Instance Type Name", + "Instance Properties", + "Anchor GUID", + "Is Own Anchor", + "Parent GUID", + "Parent Relationship Type Name", + "Parent at End1" + ], + "Journal Entry": "Added 2026-11-12 -- MetadataExpert family (instance commands).", + "idempotent": false, + "chat_allowed": false + } + } +} diff --git a/md_processing/data/generated_format_sets.json b/md_processing/data/generated_format_sets.json index e69af05c..42cbe105 100644 --- a/md_processing/data/generated_format_sets.json +++ b/md_processing/data/generated_format_sets.json @@ -200035,6 +200035,1824 @@ "get_additional_props": null, "question_spec": null }, + "Type-Def-DrE-Basic": { + "target_type": "Type Def", + "heading": "Type-Def-DrE-Basic Attributes", + "description": "Auto-generated format for Type Def (Create, Basic).", + "aliases": [], + "annotations": {}, + "family": "MetadataExpert", + "formats": [ + { + "types": [ + "LIST" + ], + "attributes": [ + { + "name": "Type Def Name", + "key": "type_def_name", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [] + }, + { + "name": "Type Def Category", + "key": "type_def_category", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [ + "Entity", + "Relationship", + "Classification" + ] + }, + { + "name": "Super Type", + "key": "super_type", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [] + }, + { + "name": "Type Def Description", + "key": "type_def_description", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [] + }, + { + "name": "Type Def Attributes", + "key": "type_def_attributes", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [] + }, + { + "name": "Valid Entity Types", + "key": "valid_entity_types", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [] + }, + { + "name": "Propagatable", + "key": "propagatable", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [] + }, + { + "name": "End1 Entity Type", + "key": "end1_entity_type", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [] + }, + { + "name": "End1 Attribute Name", + "key": "end1_attribute_name", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [] + }, + { + "name": "End2 Entity Type", + "key": "end2_entity_type", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [] + }, + { + "name": "End2 Attribute Name", + "key": "end2_attribute_name", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [] + } + ], + "columns": [ + { + "name": "Type Def Name", + "key": "type_def_name", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + }, + { + "name": "Type Def Category", + "key": "type_def_category", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [ + "Entity", + "Relationship", + "Classification" + ], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + }, + { + "name": "Super Type", + "key": "super_type", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + }, + { + "name": "Type Def Description", + "key": "type_def_description", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + }, + { + "name": "Type Def Attributes", + "key": "type_def_attributes", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + }, + { + "name": "Valid Entity Types", + "key": "valid_entity_types", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + }, + { + "name": "Propagatable", + "key": "propagatable", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + }, + { + "name": "End1 Entity Type", + "key": "end1_entity_type", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + }, + { + "name": "End1 Attribute Name", + "key": "end1_attribute_name", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + }, + { + "name": "End2 Entity Type", + "key": "end2_entity_type", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + }, + { + "name": "End2 Attribute Name", + "key": "end2_attribute_name", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + } + ] + }, + { + "types": [ + "ALL" + ], + "attributes": [ + { + "name": "Type Def Name", + "key": "type_def_name", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [] + }, + { + "name": "Type Def Category", + "key": "type_def_category", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [ + "Entity", + "Relationship", + "Classification" + ] + }, + { + "name": "Super Type", + "key": "super_type", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [] + }, + { + "name": "Type Def Description", + "key": "type_def_description", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [] + }, + { + "name": "Type Def Attributes", + "key": "type_def_attributes", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [] + }, + { + "name": "Valid Entity Types", + "key": "valid_entity_types", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [] + }, + { + "name": "Propagatable", + "key": "propagatable", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [] + }, + { + "name": "End1 Entity Type", + "key": "end1_entity_type", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [] + }, + { + "name": "End1 Attribute Name", + "key": "end1_attribute_name", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [] + }, + { + "name": "End2 Entity Type", + "key": "end2_entity_type", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [] + }, + { + "name": "End2 Attribute Name", + "key": "end2_attribute_name", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [] + } + ], + "columns": [ + { + "name": "Type Def Name", + "key": "type_def_name", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + }, + { + "name": "Type Def Category", + "key": "type_def_category", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [ + "Entity", + "Relationship", + "Classification" + ], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + }, + { + "name": "Super Type", + "key": "super_type", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + }, + { + "name": "Type Def Description", + "key": "type_def_description", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + }, + { + "name": "Type Def Attributes", + "key": "type_def_attributes", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + }, + { + "name": "Valid Entity Types", + "key": "valid_entity_types", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + }, + { + "name": "Propagatable", + "key": "propagatable", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + }, + { + "name": "End1 Entity Type", + "key": "end1_entity_type", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + }, + { + "name": "End1 Attribute Name", + "key": "end1_attribute_name", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + }, + { + "name": "End2 Entity Type", + "key": "end2_entity_type", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + }, + { + "name": "End2 Attribute Name", + "key": "end2_attribute_name", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + } + ] + } + ], + "action": null, + "get_additional_props": null, + "question_spec": null + }, + "Type-Def-DrE-Advanced": { + "target_type": "Type Def", + "heading": "Type-Def-DrE-Advanced Attributes", + "description": "Auto-generated format for Type Def (Create, Advanced).", + "aliases": [], + "annotations": {}, + "family": "MetadataExpert", + "formats": [ + { + "types": [ + "LIST" + ], + "attributes": [ + { + "name": "Type Def Name", + "key": "type_def_name", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [] + }, + { + "name": "Type Def Category", + "key": "type_def_category", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [ + "Entity", + "Relationship", + "Classification" + ] + }, + { + "name": "Super Type", + "key": "super_type", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [] + }, + { + "name": "Type Def Description", + "key": "type_def_description", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [] + }, + { + "name": "Type Def Attributes", + "key": "type_def_attributes", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [] + }, + { + "name": "Valid Entity Types", + "key": "valid_entity_types", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [] + }, + { + "name": "Propagatable", + "key": "propagatable", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [] + }, + { + "name": "End1 Entity Type", + "key": "end1_entity_type", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [] + }, + { + "name": "End1 Attribute Name", + "key": "end1_attribute_name", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [] + }, + { + "name": "End2 Entity Type", + "key": "end2_entity_type", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [] + }, + { + "name": "End2 Attribute Name", + "key": "end2_attribute_name", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [] + } + ], + "columns": [ + { + "name": "Type Def Name", + "key": "type_def_name", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + }, + { + "name": "Type Def Category", + "key": "type_def_category", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [ + "Entity", + "Relationship", + "Classification" + ], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + }, + { + "name": "Super Type", + "key": "super_type", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + }, + { + "name": "Type Def Description", + "key": "type_def_description", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + }, + { + "name": "Type Def Attributes", + "key": "type_def_attributes", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + }, + { + "name": "Valid Entity Types", + "key": "valid_entity_types", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + }, + { + "name": "Propagatable", + "key": "propagatable", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + }, + { + "name": "End1 Entity Type", + "key": "end1_entity_type", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + }, + { + "name": "End1 Attribute Name", + "key": "end1_attribute_name", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + }, + { + "name": "End2 Entity Type", + "key": "end2_entity_type", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + }, + { + "name": "End2 Attribute Name", + "key": "end2_attribute_name", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + } + ] + }, + { + "types": [ + "ALL" + ], + "attributes": [ + { + "name": "Type Def Name", + "key": "type_def_name", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [] + }, + { + "name": "Type Def Category", + "key": "type_def_category", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [ + "Entity", + "Relationship", + "Classification" + ] + }, + { + "name": "Super Type", + "key": "super_type", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [] + }, + { + "name": "Type Def Description", + "key": "type_def_description", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [] + }, + { + "name": "Type Def Attributes", + "key": "type_def_attributes", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [] + }, + { + "name": "Valid Entity Types", + "key": "valid_entity_types", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [] + }, + { + "name": "Propagatable", + "key": "propagatable", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [] + }, + { + "name": "End1 Entity Type", + "key": "end1_entity_type", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [] + }, + { + "name": "End1 Attribute Name", + "key": "end1_attribute_name", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [] + }, + { + "name": "End2 Entity Type", + "key": "end2_entity_type", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [] + }, + { + "name": "End2 Attribute Name", + "key": "end2_attribute_name", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [] + } + ], + "columns": [ + { + "name": "Type Def Name", + "key": "type_def_name", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + }, + { + "name": "Type Def Category", + "key": "type_def_category", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [ + "Entity", + "Relationship", + "Classification" + ], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + }, + { + "name": "Super Type", + "key": "super_type", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + }, + { + "name": "Type Def Description", + "key": "type_def_description", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + }, + { + "name": "Type Def Attributes", + "key": "type_def_attributes", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + }, + { + "name": "Valid Entity Types", + "key": "valid_entity_types", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + }, + { + "name": "Propagatable", + "key": "propagatable", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + }, + { + "name": "End1 Entity Type", + "key": "end1_entity_type", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + }, + { + "name": "End1 Attribute Name", + "key": "end1_attribute_name", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + }, + { + "name": "End2 Entity Type", + "key": "end2_entity_type", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + }, + { + "name": "End2 Attribute Name", + "key": "end2_attribute_name", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + } + ] + } + ], + "action": null, + "get_additional_props": null, + "question_spec": null + }, + "Enum-Def-DrE-Basic": { + "target_type": "Enum Def", + "heading": "Enum-Def-DrE-Basic Attributes", + "description": "Auto-generated format for Enum Def (Create, Basic).", + "aliases": [], + "annotations": {}, + "family": "MetadataExpert", + "formats": [ + { + "types": [ + "LIST" + ], + "attributes": [ + { + "name": "Enum Def Name", + "key": "enum_def_name", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [] + }, + { + "name": "Enum Def Description", + "key": "enum_def_description", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [] + }, + { + "name": "Enum Values", + "key": "enum_values", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [] + }, + { + "name": "Enum Default Value", + "key": "enum_default_value", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [] + } + ], + "columns": [ + { + "name": "Enum Def Name", + "key": "enum_def_name", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + }, + { + "name": "Enum Def Description", + "key": "enum_def_description", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + }, + { + "name": "Enum Values", + "key": "enum_values", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + }, + { + "name": "Enum Default Value", + "key": "enum_default_value", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + } + ] + }, + { + "types": [ + "ALL" + ], + "attributes": [ + { + "name": "Enum Def Name", + "key": "enum_def_name", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [] + }, + { + "name": "Enum Def Description", + "key": "enum_def_description", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [] + }, + { + "name": "Enum Values", + "key": "enum_values", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [] + }, + { + "name": "Enum Default Value", + "key": "enum_default_value", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [] + } + ], + "columns": [ + { + "name": "Enum Def Name", + "key": "enum_def_name", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + }, + { + "name": "Enum Def Description", + "key": "enum_def_description", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + }, + { + "name": "Enum Values", + "key": "enum_values", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + }, + { + "name": "Enum Default Value", + "key": "enum_default_value", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + } + ] + } + ], + "action": null, + "get_additional_props": null, + "question_spec": null + }, + "Enum-Def-DrE-Advanced": { + "target_type": "Enum Def", + "heading": "Enum-Def-DrE-Advanced Attributes", + "description": "Auto-generated format for Enum Def (Create, Advanced).", + "aliases": [], + "annotations": {}, + "family": "MetadataExpert", + "formats": [ + { + "types": [ + "LIST" + ], + "attributes": [ + { + "name": "Enum Def Name", + "key": "enum_def_name", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [] + }, + { + "name": "Enum Def Description", + "key": "enum_def_description", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [] + }, + { + "name": "Enum Values", + "key": "enum_values", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [] + }, + { + "name": "Enum Default Value", + "key": "enum_default_value", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [] + } + ], + "columns": [ + { + "name": "Enum Def Name", + "key": "enum_def_name", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + }, + { + "name": "Enum Def Description", + "key": "enum_def_description", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + }, + { + "name": "Enum Values", + "key": "enum_values", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + }, + { + "name": "Enum Default Value", + "key": "enum_default_value", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + } + ] + }, + { + "types": [ + "ALL" + ], + "attributes": [ + { + "name": "Enum Def Name", + "key": "enum_def_name", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [] + }, + { + "name": "Enum Def Description", + "key": "enum_def_description", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [] + }, + { + "name": "Enum Values", + "key": "enum_values", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [] + }, + { + "name": "Enum Default Value", + "key": "enum_default_value", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [] + } + ], + "columns": [ + { + "name": "Enum Def Name", + "key": "enum_def_name", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + }, + { + "name": "Enum Def Description", + "key": "enum_def_description", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + }, + { + "name": "Enum Values", + "key": "enum_values", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + }, + { + "name": "Enum Default Value", + "key": "enum_default_value", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + } + ] + } + ], + "action": null, + "get_additional_props": null, + "question_spec": null + }, + "Instance-DrE-Basic": { + "target_type": "Instance", + "heading": "Instance-DrE-Basic Attributes", + "description": "Auto-generated format for Instance (Create, Basic).", + "aliases": [], + "annotations": {}, + "family": "MetadataExpert", + "formats": [ + { + "types": [ + "LIST" + ], + "attributes": [ + { + "name": "Instance Type Name", + "key": "instance_type_name", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [] + }, + { + "name": "Instance Properties", + "key": "instance_properties", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [] + }, + { + "name": "Anchor GUID", + "key": "anchor_guid", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [] + }, + { + "name": "Is Own Anchor", + "key": "is_own_anchor", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [] + }, + { + "name": "Parent GUID", + "key": "parent_guid", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [] + } + ], + "columns": [ + { + "name": "Instance Type Name", + "key": "instance_type_name", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + }, + { + "name": "Instance Properties", + "key": "instance_properties", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + }, + { + "name": "Anchor GUID", + "key": "anchor_guid", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + }, + { + "name": "Is Own Anchor", + "key": "is_own_anchor", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + }, + { + "name": "Parent GUID", + "key": "parent_guid", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + } + ] + }, + { + "types": [ + "ALL" + ], + "attributes": [ + { + "name": "Instance Type Name", + "key": "instance_type_name", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [] + }, + { + "name": "Instance Properties", + "key": "instance_properties", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [] + }, + { + "name": "Anchor GUID", + "key": "anchor_guid", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [] + }, + { + "name": "Is Own Anchor", + "key": "is_own_anchor", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [] + }, + { + "name": "Parent GUID", + "key": "parent_guid", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [] + } + ], + "columns": [ + { + "name": "Instance Type Name", + "key": "instance_type_name", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + }, + { + "name": "Instance Properties", + "key": "instance_properties", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + }, + { + "name": "Anchor GUID", + "key": "anchor_guid", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + }, + { + "name": "Is Own Anchor", + "key": "is_own_anchor", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + }, + { + "name": "Parent GUID", + "key": "parent_guid", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + } + ] + } + ], + "action": null, + "get_additional_props": null, + "question_spec": null + }, + "Instance-DrE-Advanced": { + "target_type": "Instance", + "heading": "Instance-DrE-Advanced Attributes", + "description": "Auto-generated format for Instance (Create, Advanced).", + "aliases": [], + "annotations": {}, + "family": "MetadataExpert", + "formats": [ + { + "types": [ + "LIST" + ], + "attributes": [ + { + "name": "Instance Type Name", + "key": "instance_type_name", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [] + }, + { + "name": "Instance Properties", + "key": "instance_properties", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [] + }, + { + "name": "Anchor GUID", + "key": "anchor_guid", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [] + }, + { + "name": "Is Own Anchor", + "key": "is_own_anchor", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [] + }, + { + "name": "Parent GUID", + "key": "parent_guid", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [] + } + ], + "columns": [ + { + "name": "Instance Type Name", + "key": "instance_type_name", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + }, + { + "name": "Instance Properties", + "key": "instance_properties", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + }, + { + "name": "Anchor GUID", + "key": "anchor_guid", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + }, + { + "name": "Is Own Anchor", + "key": "is_own_anchor", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + }, + { + "name": "Parent GUID", + "key": "parent_guid", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + } + ] + }, + { + "types": [ + "ALL" + ], + "attributes": [ + { + "name": "Instance Type Name", + "key": "instance_type_name", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [] + }, + { + "name": "Instance Properties", + "key": "instance_properties", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [] + }, + { + "name": "Anchor GUID", + "key": "anchor_guid", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [] + }, + { + "name": "Is Own Anchor", + "key": "is_own_anchor", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [] + }, + { + "name": "Parent GUID", + "key": "parent_guid", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [] + } + ], + "columns": [ + { + "name": "Instance Type Name", + "key": "instance_type_name", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + }, + { + "name": "Instance Properties", + "key": "instance_properties", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + }, + { + "name": "Anchor GUID", + "key": "anchor_guid", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + }, + { + "name": "Is Own Anchor", + "key": "is_own_anchor", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + }, + { + "name": "Parent GUID", + "key": "parent_guid", + "value": null, + "format": false, + "detail_spec": null, + "valid_values": [], + "render": null, + "max_items": null, + "max_lines": null, + "label": null + } + ] + } + ], + "action": null, + "get_additional_props": null, + "question_spec": null + }, "Study-Project-DrE-Basic": { "target_type": "Study Project", "heading": "Study-Project-DrE-Basic Attributes", diff --git a/md_processing/dr_egeria.py b/md_processing/dr_egeria.py index 0019bdd0..0c7370f2 100644 --- a/md_processing/dr_egeria.py +++ b/md_processing/dr_egeria.py @@ -56,6 +56,10 @@ SubscriptionTypeProcessor, BitolDocumentProcessor, SchemaElementProcessor, SchemaTemplateProcessor, SchemaLinkProcessor, SchemaClassificationProcessor, ) +from md_processing.v2.metadata_expert import ( + TypeDefProcessor, EnumDefProcessor, CountMetadataElementsProcessor, PublishMetadataElementProcessor, + InstanceProcessor, InstanceRelationshipProcessor, InstanceClassificationProcessor, +) from pyegeria import settings, EgeriaTech, PyegeriaException, print_basic_exception, print_validation_error from pyegeria.core.config import settings, resolve_egeria_credentials @@ -432,6 +436,21 @@ def reg(base, cls): reg("Initiate Subscription", InitiateSubscriptionProcessor) reg("Cancel Subscription", CancelSubscriptionProcessor) + # MetadataExpert (Advanced): type system maintenance + raw element count/publish (metadata_expert.py) + for _verb in ("Create", "Update", "Delete"): + reg(f"{_verb} Type Def", TypeDefProcessor) + for _verb in ("Create", "Delete"): + reg(f"{_verb} Enum Def", EnumDefProcessor) + reg("Count Metadata Elements", CountMetadataElementsProcessor) + reg("Publish Metadata Element", PublishMetadataElementProcessor) + for _verb in ("Create", "Update", "Delete"): + reg(f"{_verb} Instance", InstanceProcessor) + reg("Link Instances", InstanceRelationshipProcessor) + reg("Unlink Instances", InstanceRelationshipProcessor) + reg("Update Relationship", InstanceRelationshipProcessor) + for _verb in ("Classify", "Reclassify", "Declassify"): + reg(f"{_verb} Instance", InstanceClassificationProcessor) + # Digital Products: subscription types and ODCS/ODPS Bitol documents (product_manager.py) reg("Create Subscription Type", SubscriptionTypeProcessor) for _bitol_verb in ("Import", "Publish"): diff --git a/md_processing/md_processing_utils/compact_loader.py b/md_processing/md_processing_utils/compact_loader.py index 16c70f14..08505850 100644 --- a/md_processing/md_processing_utils/compact_loader.py +++ b/md_processing/md_processing_utils/compact_loader.py @@ -27,6 +27,8 @@ def _command_to_spec(name: str, cdef: Dict[str, Any], expanded: Dict[str, Any]) "verb", "upsert", "attach", + "idempotent", # False: running the command again makes another element/relationship + "chat_allowed", # False: a chat agent must not compose this command "ReferenceURL", "level", "find_method", diff --git a/md_processing/v2/additional_properties.py b/md_processing/v2/additional_properties.py new file mode 100644 index 00000000..13ea1eb1 --- /dev/null +++ b/md_processing/v2/additional_properties.py @@ -0,0 +1,63 @@ +"""additionalProperties on a merge update: keep the keys the document doesn't write. + +Egeria replaces a map property as one value, even on a merge update (`mergeUpdate: true` merges +properties, not the keys inside one map property). A Dr.Egeria Create that finds its element (and +becomes an Update), or an Update with Merge Update true, used to send only the keys its document +gave, so every other key on the element was removed: on laz a SavedQuery's refresh keys +(lastRefreshedTime, resultCount, resultsSetGuid, smartQueryRelGuid, lastRefreshParams) were wiped +this way, and a CDO Report's search keys nearly were. + +The fix: on a merge update, read the element's stored map and overlay the document's keys on it, +key by key, so the map sent is the whole map. A replace update (Merge Update false) still replaces, +and says which keys go. + +There is no way in a Dr.Egeria document to clear a single key: a Dictionary value is parsed to +strings (a JSON `null` becomes the string "null"), and an empty value is no value. To remove keys, +use Merge Update false, which replaces the element's properties with exactly what the document gives. +""" + +from __future__ import annotations + +from typing import Any, Dict, Iterable, Optional + + +class AdditionalPropertiesUnreadable(ValueError): + """A merge update would send additionalProperties, but the stored map couldn't be read.""" + + +def stored_additional_properties(element: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]: + """The element's stored additionalProperties ({} when it has none), or None when the element + wasn't read with its properties (so its map is unknown).""" + if not isinstance(element, dict): + return None + properties = element.get("properties") + if not isinstance(properties, dict): + return None + stored = properties.get("additionalProperties") + if stored is None: + return {} + return dict(stored) if isinstance(stored, dict) else None + + +def merge_additional_properties(stored: Dict[str, Any], document: Optional[Dict[str, Any]]) -> Dict[str, Any]: + """The stored map with the document's keys laid over it: the document wins key by key, and keys + it doesn't give are kept.""" + merged = dict(stored or {}) + merged.update(document or {}) + return merged + + +def replace_note(stored: Optional[Dict[str, Any]], sent_keys: Iterable[str]) -> Optional[str]: + """The sentence a replace update (Merge Update false) adds, naming the stored keys it removes.""" + removed = sorted(set(stored or {}) - set(sent_keys)) + if not removed: + return None + return ("Merge Update is false, so this update replaces additionalProperties as a whole: " + f"these stored keys are removed: {', '.join(removed)}") + + +def unreadable_message(what: str) -> str: + return (f"Refusing to update {what}: the update would send additionalProperties, but the element's " + "stored additionalProperties couldn't be read, and Egeria replaces that map as a whole, so the " + "keys this document doesn't give would be removed. Re-run once the element can be read, or set " + "Merge Update to false to replace the map on purpose.") diff --git a/md_processing/v2/dashboard_sheet.py b/md_processing/v2/dashboard_sheet.py index c79c4d6d..57625035 100644 --- a/md_processing/v2/dashboard_sheet.py +++ b/md_processing/v2/dashboard_sheet.py @@ -414,8 +414,11 @@ async def list_fields(self, container_guid: Optional[str]) -> List[FieldInfo]: # ------------------------------------------------------------ sheet async def ensure_sheet(self, sheet_name: str, heading: str, description: str = "", - family: Optional[str] = None, aliases: Optional[List[str]] = None) -> SheetChain: - """Create or update the Dashboard, its DisplayDataSchemaType and the sheet's DisplayDataContainer.""" + family: Optional[str] = None, aliases: Optional[List[str]] = None, + additional: Optional[Dict[str, Any]] = None) -> SheetChain: + """Create or update the Dashboard, its DisplayDataSchemaType and the sheet's DisplayDataContainer. + `additional`: the document's own Additional Properties for the Dashboard (e.g. pyegeria.authoring's + publication stamp), merged with the stored map like every other key; `family` wins its own key.""" chain = await self.find_chain(sheet_name) # 1. the Dashboard asset -- through upsert_dashboard_asset, the same code as `Create Dashboard` @@ -426,7 +429,8 @@ async def ensure_sheet(self, sheet_name: str, heading: str, description: str = " "displayName": heading or sheet_name, "description": description or None, "aliases": list(aliases) if aliases else None, - "additionalProperties": {"family": family} if family else None, + "additionalProperties": ({**{str(k): str(v) for k, v in (additional or {}).items() if v is not None}, + **({"family": family} if family else {})} or None), } if self.dry_run: if chain.dashboard is None: @@ -828,17 +832,20 @@ async def apply_changes(self) -> str: heading = attributes.get("Dashboard Sheet Heading", {}).get("value") or name description = attributes.get("Dashboard Sheet Description", {}).get("value") or "" family = attributes.get("Dashboard Sheet Family", {}).get("value") or None + additional = attributes.get("Additional Properties", {}).get("value") + additional = additional if isinstance(additional, dict) else None # A sheet keeps its heading, description, family, aliases and owner (locally and on its Dashboard); # the bundle attributes the command also offers are reported, not dropped (F-B group 5). from md_processing.md_processing_utils.common_md_utils import BUNDLE_PROPERTIES from md_processing.md_processing_utils.typed_properties import record_not_applied - for attribute in [*BUNDLE_PROPERTIES, "Status", "Content Status", "Purpose", "Parent Relationship Attributes"]: + for attribute in [*(a for a in BUNDLE_PROPERTIES if a != "Additional Properties"), "Status", "Content Status", + "Purpose", "Parent Relationship Attributes"]: entry = attributes.get(attribute) or {} if entry.get("value") not in (None, "", [], {}) and not entry.get("is_default"): if attribute == "Description" and attributes.get("Dashboard Sheet Description", {}).get("value"): note = "the sheet's description is Dashboard Sheet Description" else: - note = "a Dashboard Sheet keeps only its heading, description, family, aliases and owner" + note = "a Dashboard Sheet keeps only its heading, description, family, aliases, owner and Additional Properties" record_not_applied(attributes, attribute, [f"not applied: {attribute}: {note}"]) path = _default_store_path() @@ -852,7 +859,7 @@ async def apply_changes(self) -> str: ownership = {} async def _work(writer: DashboardSheetEgeriaWriter): - chain = await writer.ensure_sheet(name, heading, description, family, aliases) + chain = await writer.ensure_sheet(name, heading, description, family, aliases, additional) self.parsed_output["guid"] = chain.dashboard_guid if owner: ownership["status"] = await writer.ensure_ownership(chain.dashboard_guid, dashboard_qn(name), owner) diff --git a/md_processing/v2/extraction.py b/md_processing/v2/extraction.py index d41cf62d..dd92588b 100644 --- a/md_processing/v2/extraction.py +++ b/md_processing/v2/extraction.py @@ -39,6 +39,8 @@ # integration daemon). Without these the `## Import ...` / `## Publish ...` headings never match # cmd_header_rx and silently fall through as prose -- no error, no summary row. "Import", "Publish", + # MetadataExpert family: "Count Metadata Elements". + "Count", } # Commands that appear as a single word (no separate verb) and map to canonical verb+object diff --git a/md_processing/v2/metadata_expert.py b/md_processing/v2/metadata_expert.py new file mode 100644 index 00000000..831916b4 --- /dev/null +++ b/md_processing/v2/metadata_expert.py @@ -0,0 +1,571 @@ +""" +MetadataExpert processors for Dr.Egeria v2 (Advanced commands only). + +Maintenance of Egeria's open metadata *type system* plus two raw metadata-element operations, all backed +by ``pyegeria/omvs/metadata_expert.py``: + +- Create Type Def -> add_type_def (entity, relationship or classification definition) +- Update Type Def -> update_type_def (Egeria rejects every patch today: ISSUE-124) +- Delete Type Def -> delete_type_def +- Create Enum Def -> add_enum_def +- Delete Enum Def -> delete_enum_def (Egeria returns a 500: ISSUE-125; enums cannot be removed) +- Count Metadata Elements -> count_metadata_elements +- Publish Metadata Element -> publish_metadata_element +- Create/Update/Delete Instance -> create/update_metadata_element_properties/delete_metadata_element +- Link/Unlink Instances, Update Relationship -> create/delete/update_related_elements* +- Classify/Reclassify/Declassify Instance -> classify/reclassify/declassify_metadata_element + +Instance, relationship and classification properties are sent as typed property maps +(EntityElementProperties / RelationshipElementProperties / ClassificationElementProperties with a +propertyValueMap of PrimitiveTypePropertyValue), the shape shown in the SDK docstrings. NOT verified +against a live server (Egeria was read-only when these were written). + +Like the Automation family these are actions, not create-or-update of a named element: no as-is lookup +and no Create<->Update rewrite. +""" +import re +from datetime import datetime +from typing import Any, Dict, List, Optional + +from loguru import logger + +from pyegeria import PyegeriaException +from md_processing.v2.processors import AsyncBaseCommandProcessor + +PRIMITIVE_TYPES = {"string", "int", "long", "boolean", "date", "float", "double", "short", "byte", "char", + "bigint", "bigdecimal", "object"} +_CATEGORY_CLASS = {"entity": "OpenMetadataEntityDef", "relationship": "OpenMetadataRelationshipDef", + "classification": "OpenMetadataClassificationDef"} + + +def _v(attributes: Dict[str, Any], name: str, default=None): + value = attributes.get(name, {}).get("value", default) + return default if value in (None, "") else value + + +def _names(value: Any) -> List[str]: + """A list-ish string (newline, comma or bullet separated) or list -> clean names.""" + if value is None: + return [] + items = value if isinstance(value, list) else re.split(r"[\n,]", str(value)) + return [str(i).strip().lstrip("-*• ").strip() for i in items if str(i).strip().lstrip("-*• ").strip()] + + +def _bool(value: Any) -> Optional[bool]: + if isinstance(value, bool) or value is None: + return value + text = str(value).strip().lower() + return True if text in ("true", "yes", "y", "1") else False if text in ("false", "no", "n", "0") else None + + +def confirm_name(attributes: Dict[str, Any], name_attr: str, confirm_attr: str, what: str) -> str: + """Platform-wide type system changes are refused unless the confirmation repeats the name exactly + (case-sensitive, surrounding spaces ignored). Called before anything is sent to Egeria.""" + name = str(_v(attributes, name_attr, "")).strip() + if not name: + raise PyegeriaException(f"{name_attr} is required.") + if str(_v(attributes, confirm_attr, "")).strip() != name: + raise PyegeriaException( + f"{what} changes the type system for every user and app on this platform. Repeat the name ({name}) " + f"in {confirm_attr} to proceed; nothing was changed.") + return name + + +# (name attribute, confirmation attribute) for the commands that change the type system +CONFIRMATIONS = (("Type Def Name", "Confirm Type Def Name"), ("Enum Def Name", "Confirm Enum Def Name")) + + +def _link(name: str) -> Dict[str, str]: + return {"name": name} + + +def _attribute_type(type_name: str) -> Dict[str, str]: + cls = "OpenMetadataPrimitiveDef" if type_name.lower() in PRIMITIVE_TYPES else "OpenMetadataEnumDef" + return {"class": cls, "name": type_name} + + +def attribute_definitions(attributes: Dict[str, Any]) -> List[Dict[str, Any]]: + """'Type Def Attributes' Dictionary (attributeName: typeName) -> attributeDefinitions.""" + return [{"attributeName": str(name), "attributeType": _attribute_type(str(type_name).strip())} + for name, type_name in (_v(attributes, "Type Def Attributes") or {}).items()] + + +def type_def_body(attributes: Dict[str, Any]) -> Dict[str, Any]: + category = str(_v(attributes, "Type Def Category", "Entity")).strip().lower() + if category not in _CATEGORY_CLASS: + raise PyegeriaException(f"Type Def Category must be Entity, Relationship or Classification, not '{category}'.") + body: Dict[str, Any] = {"class": _CATEGORY_CLASS[category], "name": _v(attributes, "Type Def Name")} + if _v(attributes, "Type Def Description"): + body["description"] = _v(attributes, "Type Def Description") + if _v(attributes, "Super Type"): + body["superType"] = _link(_v(attributes, "Super Type")) + if attribute_definitions(attributes): + body["attributeDefinitions"] = attribute_definitions(attributes) + if category == "classification": + valid = _names(_v(attributes, "Valid Entity Types")) + if valid: + body["validEntityDefs"] = [_link(n) for n in valid] + if _bool(_v(attributes, "Propagatable")) is not None: + body["propagatable"] = _bool(_v(attributes, "Propagatable")) + elif category == "relationship": + # Literal attribute names (not f-strings) so the attribute-consumption audit can see them. + for key, entity_attr, name_attr in (("endDef1", "End1 Entity Type", "End1 Attribute Name"), + ("endDef2", "End2 Entity Type", "End2 Attribute Name")): + entity, attr_name = _v(attributes, entity_attr), _v(attributes, name_attr) + if not entity or not attr_name: + raise PyegeriaException(f"A relationship definition needs {entity_attr} and {name_attr}.") + body[key] = {"entityType": _link(entity), "attributeName": attr_name} + return body + + +def enum_def_body(attributes: Dict[str, Any]) -> Dict[str, Any]: + values = _v(attributes, "Enum Values") or {} + if not values: + raise PyegeriaException("Enum Values is required: one 'value: description' per line.") + elements = [{"ordinal": i, "value": str(value), "description": str(description or "")} + for i, (value, description) in enumerate(values.items())] + default = _v(attributes, "Enum Default Value") or elements[0]["value"] + matches = [e for e in elements if e["value"] == default] + if not matches: + raise PyegeriaException(f"Enum Default Value '{default}' is not one of the Enum Values.") + body: Dict[str, Any] = {"class": "OpenMetadataEnumDef", "name": _v(attributes, "Enum Def Name"), + "elementDefs": elements, "defaultValue": matches[0]} + if _v(attributes, "Enum Def Description"): + body["description"] = _v(attributes, "Enum Def Description") + return body + + +class MetadataExpertProcessor(AsyncBaseCommandProcessor): + """Shared behaviour: actions with no as-is element and no upsert rewrite.""" + + def supports_target_element_lookup(self) -> bool: + return False + + async def fetch_as_is(self) -> Optional[Dict[str, Any]]: + return None + + @property + def expert(self): + return self.client.metadata_expert + + def spec_flag(self, name: str, default: Any = None) -> Any: + """A command-level flag from the compact spec (idempotent, chat_allowed).""" + return (self.get_command_spec() or {}).get(name, default) + + async def validate_only(self) -> str: + report = await super().validate_only() + notes = [] + if self.spec_flag("idempotent", True) is False: + notes.append("This command is **not idempotent**: it acts on Egeria every time it runs, so running the " + "document again repeats it (Create Instance and Link Instances make a new element or " + "relationship each time). Do not put it in a re-run or heal batch.") + if self.spec_flag("chat_allowed", True) is False: + notes.append("Advanced command, not for chat agents.") + attributes = (self.parsed_output or {}).get("attributes", {}) + for name_attr, confirm_attr in CONFIRMATIONS: + if confirm_attr in (self.get_command_spec() or {}).get("custom_attributes", []): + given = str(_v(attributes, confirm_attr, "")).strip() + expected = str(_v(attributes, name_attr, "")).strip() + state = ("present and matches" if given and given == expected + else "missing" if not given else "does NOT match, so the command will be refused") + notes.append(f"**Platform-wide change.** {confirm_attr} must repeat {name_attr} exactly; it is {state}. " + f"Validate mode writes nothing to Egeria.") + guid_attr = name_attr.replace("Name", "GUID") + supplied = _v(attributes, guid_attr) + if supplied and given: + problem = await self.guid_mismatch(supplied, confirm_attr, + f"{self.command.verb} {self.command.object_type}", + enum=name_attr.startswith("Enum")) + notes.append(f"**{guid_attr} check failed:** {problem}" if problem + else f"{guid_attr} {supplied} belongs to '{given}' (read-only check).") + return report + ("\n\n> " + "\n> ".join(notes) + "\n" if notes else "") + + async def declared_properties(self, type_name: str) -> Dict[str, Any]: + """property name -> (attribute type name, category) for a type and its super types.""" + declared: Dict[str, Any] = {} + seen = set() + current: Optional[str] = type_name + while current and current not in seen: + seen.add(current) + definition = await self._type_def(current) + for attribute in definition.get("attributeDefinitions") or []: + attr_type = attribute.get("attributeType") or {} + declared.setdefault(attribute.get("attributeName"), (attr_type.get("name"), attr_type.get("category"))) + current = (definition.get("superType") or {}).get("name") + return declared + + async def type_name_of(self, result: Any, header_key: str, what: str) -> str: + """The type name in an element/relationship read back from Egeria.""" + node = result[0] if isinstance(result, list) and result else result + if isinstance(node, dict): + for candidate in (node.get(header_key), node): + name = ((candidate or {}).get("type") or {}).get("typeName") + if name: + return name + raise PyegeriaException(f"Could not read the type of {what} to check its properties.") + + def _result(self, what: str, detail: str = "") -> str: + logger.success(f"{self.command.verb} {self.command.object_type}: {what}") + text = f"\n\n## {self.command.verb} {self.command.object_type}\n\n{what}" + return f"{text}: `{detail}`\n" if detail else f"{text}\n" + + async def _type_def(self, name: str) -> Dict[str, Any]: + found = await self.client.valid_metadata._async_get_typedef_by_name(name) + if not isinstance(found, dict) or not found.get("guid"): + raise PyegeriaException(f"Type definition '{name}' was not found.") + return found + + async def type_name_for_guid(self, guid: str, enum: bool = False) -> Optional[str]: + """Name of the type or enum definition with this GUID, or None when it cannot be found. + + Egeria has no read-by-GUID for type definitions, so this lists them (reads only) and matches.""" + valid = self.client.valid_metadata + if enum: + sources = [valid._async_get_attribute_types(category="ENUM")] + else: + sources = [valid._async_get_all_entity_defs(), valid._async_get_all_relationship_defs(), + valid._async_get_all_classification_defs()] + for source in sources: + found = await source + for item in found if isinstance(found, list) else []: + if isinstance(item, dict) and item.get("guid") == guid: + return item.get("name") + return None + + async def guid_mismatch(self, guid: str, echo_attr: str, what: str, enum: bool = False) -> Optional[str]: + """Why a supplied GUID must not be used with this confirmation, or None when it matches. + + A wrong GUID with the right echo would change a different definition platform-wide. Reads only.""" + echo = str(_v(self.parsed_output["attributes"], echo_attr, "")).strip() + actual = await self.type_name_for_guid(guid, enum) + if actual is None: + return (f"{what}: GUID {guid} was not found among the {'enum' if enum else 'type'} definitions, so it " + f"cannot be checked against {echo_attr} ({echo}); nothing was changed.") + if actual != echo: + return (f"{what}: GUID {guid} belongs to '{actual}', not '{echo}' ({echo_attr}). This would change a " + f"different definition for every user and app on this platform; nothing was changed.") + return None + + async def check_guid_matches_echo(self, guid: str, echo_attr: str, what: str, enum: bool = False) -> None: + problem = await self.guid_mismatch(guid, echo_attr, what, enum) + if problem: + raise PyegeriaException(problem) + + async def _enum_guid(self, name: str) -> str: + found = await self.client.valid_metadata._async_get_attribute_types(category="ENUM") + for item in found if isinstance(found, list) else []: + if isinstance(item, dict) and item.get("name") == name and item.get("guid"): + return item["guid"] + raise PyegeriaException(f"Enum definition '{name}' was not found.") + + +class TypeDefProcessor(MetadataExpertProcessor): + async def apply_changes(self) -> str: + attributes = self.parsed_output["attributes"] + name = _v(attributes, "Type Def Name") + if not name: + raise PyegeriaException("Type Def Name is required.") + verb = self.command.verb + if verb in ("Update", "Delete"): + confirm_name(attributes, "Type Def Name", "Confirm Type Def Name", f"{verb} Type Def") + if verb == "Create": + guid = await self.expert._async_add_type_def(type_def_body(attributes)) + self.parsed_output["guid"] = guid if isinstance(guid, str) else None + return self._result(f"Added type definition '{name}'", guid if isinstance(guid, str) else "") + supplied = _v(attributes, "Type Def GUID") + if supplied: + await self.check_guid_matches_echo(supplied, "Confirm Type Def Name", f"{verb} Type Def") + guid = supplied or (await self._type_def(name))["guid"] + if verb == "Delete": + await self.expert._async_delete_type_def(guid, name) + return self._result(f"Deleted type definition '{name}'", guid) + if verb == "Update": + patch = {"class": "OpenMetadataTypeDefPatch", "typeDefGUID": guid, "typeDefName": name, + "applyToVersion": int(_v(attributes, "Apply To Version"))} + if _v(attributes, "Type Def Description"): + patch["description"] = _v(attributes, "Type Def Description") + if attribute_definitions(attributes): + patch["attributeDefinitions"] = attribute_definitions(attributes) + await self.expert._async_update_type_def(patch) + return self._result(f"Updated type definition '{name}'", guid) + raise PyegeriaException(f"Unsupported verb '{verb}' for Type Def.") + + +class EnumDefProcessor(MetadataExpertProcessor): + async def apply_changes(self) -> str: + attributes = self.parsed_output["attributes"] + name = _v(attributes, "Enum Def Name") + if not name: + raise PyegeriaException("Enum Def Name is required.") + if self.command.verb == "Create": + guid = await self.expert._async_add_enum_def(enum_def_body(attributes)) + self.parsed_output["guid"] = guid if isinstance(guid, str) else None + return self._result(f"Added enum definition '{name}'", guid if isinstance(guid, str) else "") + if self.command.verb == "Delete": + confirm_name(attributes, "Enum Def Name", "Confirm Enum Def Name", "Delete Enum Def") + supplied = _v(attributes, "Enum Def GUID") + if supplied: + await self.check_guid_matches_echo(supplied, "Confirm Enum Def Name", "Delete Enum Def", enum=True) + guid = supplied or await self._enum_guid(name) + await self.expert._async_delete_enum_def(guid, name) + return self._result(f"Deleted enum definition '{name}'", guid) + raise PyegeriaException(f"Unsupported verb '{self.command.verb}' for Enum Def.") + + +class CountMetadataElementsProcessor(MetadataExpertProcessor): + async def apply_changes(self) -> str: + attributes = self.parsed_output["attributes"] + body: Dict[str, Any] = {"class": "FindRequestBody"} + if _v(attributes, "Count Element Type Name"): + body["metadataElementTypeName"] = _v(attributes, "Count Element Type Name") + if _v(attributes, "Count Search String"): + body["searchString"] = _v(attributes, "Count Search String") + count = await self.expert._async_count_metadata_elements( + body, push_down=_bool(_v(attributes, "Count Push Down"))) + return self._result("Matching metadata elements", str(count)) + + +class PublishMetadataElementProcessor(MetadataExpertProcessor): + async def apply_changes(self) -> str: + guid = self.parsed_output["attributes"].get("Metadata Element GUID", {}).get("value") + if not guid: + raise PyegeriaException("Metadata Element GUID is required.") + await self.expert._async_publish_metadata_element(guid) + self.parsed_output["guid"] = guid + return self._result("Published metadata element", guid) + + +# --- instances, relationships and classifications ------------------------------------------------- + +_INTEGER_TYPES = {"int", "long", "short", "byte", "bigint"} +_DECIMAL_TYPES = {"float", "double", "bigdecimal"} + + +def _primitive(property_name: str, value: Any, type_name: str) -> Dict[str, Any]: + """Coerce an authored value to the primitive type the type definition declares for the property. + + The declared type decides, never the text: "007" stays "007" in a string property. Anything that is + not a plain scalar is refused in words rather than flattened with str().""" + if isinstance(value, (dict, list, tuple, set)): + raise PyegeriaException( + f"Property '{property_name}' was given a map/list value; only single primitive values are supported here.") + text = str(value).strip() + kind = type_name.lower() + + def refuse(expected: str) -> PyegeriaException: + return PyegeriaException(f"Property '{property_name}' is declared {kind} but '{text}' is not {expected}.") + + if kind == "string": + parsed: Any = str(value) + elif kind == "boolean": + if text.lower() not in ("true", "false"): + raise refuse("true or false") + parsed = text.lower() == "true" + elif kind in _INTEGER_TYPES: + if not re.fullmatch(r"[+-]?\d+", text): + raise refuse("a whole number") + parsed = int(text) + elif kind in _DECIMAL_TYPES: + try: + parsed = float(text) + except ValueError: + raise refuse("a number") from None + elif kind == "char": + if len(text) != 1: + raise refuse("a single character") + parsed = text + elif kind == "date": + if re.fullmatch(r"\d+", text): + parsed = int(text) # epoch milliseconds, as Egeria stores dates + else: + try: + parsed = int(datetime.fromisoformat(text).timestamp() * 1000) + except ValueError: + raise refuse("epoch milliseconds or an ISO-8601 date") from None + else: + raise PyegeriaException(f"Property '{property_name}' has primitive type '{type_name}', which this family does not support.") + return {"class": "PrimitiveTypePropertyValue", "typeName": kind, "primitiveValue": parsed} + + +def _declared_type(property_name: str, declared: Dict[str, Any], type_name: str) -> str: + """The primitive type name for a property, or a refusal for unknown, enum, map and array properties.""" + if property_name not in declared: + raise PyegeriaException(f"'{property_name}' is not a property of {type_name} (or its super types).") + attr_type, category = declared[property_name] + category_text = str(category or "").upper() + if "ENUM" in category_text: + raise PyegeriaException( + f"Property '{property_name}' of {type_name} is an enum ({attr_type}); enum properties are not supported by this family yet.") + if "COLLECTION" in category_text or str(attr_type).lower().startswith(("map", "array")): + raise PyegeriaException( + f"Property '{property_name}' of {type_name} is a {attr_type} (map/array); not supported by this family yet.") + return str(attr_type) + + +def element_properties(properties: Optional[Dict[str, Any]], class_name: str, + declared: Optional[Dict[str, Any]] = None, type_name: str = "") -> Optional[Dict[str, Any]]: + """'name: value' Dictionary -> {"class": class_name, "propertyValueMap": {...}}; None when empty. + + declared maps each property name to (attribute type name, attribute type category), read from the + instance type's type definition and its super types. It is required whenever properties are given.""" + if not properties: + return None + if declared is None: + raise PyegeriaException(f"Cannot send properties for {type_name or 'this type'} without its type definition.") + return {"class": class_name, + "propertyValueMap": {str(k): _primitive(str(k), v, _declared_type(str(k), declared, type_name)) + for k, v in properties.items()}} + + +def _require(attributes: Dict[str, Any], name: str) -> Any: + value = _v(attributes, name) + if value is None: + raise PyegeriaException(f"{name} is required.") + return value + + +DESTRUCTIVE_DELETE_METHODS = {"PURGE", "HARD_DELETE"} + + +def _delete_method(attributes: Dict[str, Any], guid: Optional[str] = None) -> str: + """The delete method (default SOFT_DELETE). PURGE / HARD_DELETE cannot be undone, so they are refused + unless Confirm Delete GUID repeats the GUID being deleted.""" + method = str(_v(attributes, "Delete Method", "SOFT_DELETE")).strip().upper() + if method in DESTRUCTIVE_DELETE_METHODS: + confirm = str(_v(attributes, "Confirm Delete GUID", "")).strip() + if not guid or confirm != str(guid).strip(): + raise PyegeriaException( + f"{method} is permanent. Repeat the GUID being deleted ({guid}) in Confirm Delete GUID to proceed; " + f"it only works on an item that was already soft deleted.") + return method + + +def new_instance_body(attributes: Dict[str, Any], declared: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: + type_name = _require(attributes, "Instance Type Name") + body: Dict[str, Any] = {"class": "NewOpenMetadataElementRequestBody", "typeName": type_name} + props = element_properties(_v(attributes, "Instance Properties"), "EntityElementProperties", declared, type_name) + if props: + body["properties"] = props + if _v(attributes, "Anchor GUID"): + body["anchorGUID"] = _v(attributes, "Anchor GUID") + if _bool(_v(attributes, "Is Own Anchor")) is not None: + body["isOwnAnchor"] = _bool(_v(attributes, "Is Own Anchor")) + if _v(attributes, "Parent GUID"): + if not _v(attributes, "Parent Relationship Type Name"): + raise PyegeriaException("Parent Relationship Type Name is required when Parent GUID is given.") + body["parentGUID"] = _v(attributes, "Parent GUID") + body["parentRelationshipTypeName"] = _v(attributes, "Parent Relationship Type Name") + at_end_1 = _bool(_v(attributes, "Parent at End1")) + body["parentAtEnd1"] = True if at_end_1 is None else at_end_1 + return body + + +def update_properties_body(properties: Optional[Dict[str, Any]], class_name: str, + attributes: Dict[str, Any], declared: Optional[Dict[str, Any]] = None, + type_name: str = "") -> Dict[str, Any]: + props = element_properties(properties, class_name, declared, type_name) + if not props: + raise PyegeriaException("At least one property is required.") + return {"class": "UpdatePropertiesRequestBody", "properties": props, + "replaceProperties": bool(_bool(_v(attributes, "Replace Properties")))} + + +def related_elements_body(attributes: Dict[str, Any], declared: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: + type_name = _require(attributes, "Relationship Type Name") + body: Dict[str, Any] = {"class": "NewRelatedElementsRequestBody", + "typeName": type_name, + "metadataElement1GUID": _require(attributes, "Element 1 GUID"), + "metadataElement2GUID": _require(attributes, "Element 2 GUID")} + props = element_properties(_v(attributes, "Relationship Properties"), "RelationshipElementProperties", + declared, type_name) + if props: + body["properties"] = props + return body + + +class InstanceProcessor(MetadataExpertProcessor): + """Create / Update / Delete Instance.""" + + async def apply_changes(self) -> str: + attributes = self.parsed_output["attributes"] + verb = self.command.verb + if verb == "Create": + type_name = _require(attributes, "Instance Type Name") + declared = await self.declared_properties(type_name) if _v(attributes, "Instance Properties") else None + guid = await self.expert._async_create_metadata_element(new_instance_body(attributes, declared)) + self.parsed_output["guid"] = guid if isinstance(guid, str) else None + return self._result(f"Created {_v(attributes, 'Instance Type Name')} instance", guid if isinstance(guid, str) else "") + guid = _require(attributes, "Metadata Element GUID") + self.parsed_output["guid"] = guid + if verb == "Update": + type_name = await self.type_name_of( + await self.expert._async_get_metadata_element_by_guid(guid), "elementHeader", f"element {guid}") + declared = await self.declared_properties(type_name) + await self.expert._async_update_metadata_element_properties( + guid, update_properties_body(_v(attributes, "Instance Properties"), "EntityElementProperties", + attributes, declared, type_name)) + return self._result("Updated instance", guid) + if verb == "Delete": + await self.expert._async_delete_metadata_element( + guid, {"class": "DeleteElementRequestBody", "deleteMethod": _delete_method(attributes, guid)}) + return self._result("Deleted instance", guid) + raise PyegeriaException(f"Unsupported verb '{verb}' for Instance.") + + +class InstanceRelationshipProcessor(MetadataExpertProcessor): + """Link / Unlink Instances and Update Relationship.""" + + async def apply_changes(self) -> str: + attributes = self.parsed_output["attributes"] + verb, object_type = self.command.verb, self.command.object_type + if verb == "Link": + type_name = _require(attributes, "Relationship Type Name") + declared = await self.declared_properties(type_name) if _v(attributes, "Relationship Properties") else None + guid = await self.expert._async_create_related_elements(related_elements_body(attributes, declared)) + self.parsed_output["guid"] = guid if isinstance(guid, str) else None + return self._result("Created relationship", guid if isinstance(guid, str) else "") + guid = _require(attributes, "Relationship GUID") + self.parsed_output["guid"] = guid + if verb == "Unlink": + await self.expert._async_delete_related_elements( + guid, {"class": "DeleteRelationshipRequestBody", "deleteMethod": _delete_method(attributes, guid)}) + return self._result("Deleted relationship", guid) + if verb == "Update" and object_type == "Relationship": + type_name = await self.type_name_of( + await self.expert._async_get_relationship_by_guid(guid), "relationshipHeader", f"relationship {guid}") + declared = await self.declared_properties(type_name) + await self.expert._async_update_related_elements_properties( + guid, update_properties_body(_v(attributes, "Relationship Properties"), + "RelationshipElementProperties", attributes, declared, type_name)) + return self._result("Updated relationship", guid) + raise PyegeriaException(f"Unsupported command '{verb} {object_type}'.") + + +class InstanceClassificationProcessor(MetadataExpertProcessor): + """Classify / Reclassify / Declassify Instance.""" + + async def apply_changes(self) -> str: + attributes = self.parsed_output["attributes"] + guid = _require(attributes, "Metadata Element GUID") + name = _require(attributes, "Classification Name") + self.parsed_output["guid"] = guid + props = _v(attributes, "Classification Properties") + verb = self.command.verb + if verb == "Classify": + body: Dict[str, Any] = {"class": "NewClassificationRequestBody"} + declared = await self.declared_properties(name) if props else None + cls_props = element_properties(props, "ClassificationElementProperties", declared, name) + if cls_props: + body["properties"] = cls_props + await self.expert._async_classify_metadata_element(guid, name, body) + return self._result(f"Classified as {name}", guid) + if verb == "Reclassify": + declared = await self.declared_properties(name) + await self.expert._async_reclassify_metadata_element( + guid, name, update_properties_body(props, "ClassificationElementProperties", attributes, + declared, name)) + return self._result(f"Reclassified {name}", guid) + if verb == "Declassify": + await self.expert._async_declassify_metadata_element(guid, name) + return self._result(f"Removed classification {name}", guid) + raise PyegeriaException(f"Unsupported verb '{verb}' for Instance classification.") diff --git a/md_processing/v2/processors.py b/md_processing/v2/processors.py index 2d79175b..3391c739 100644 --- a/md_processing/v2/processors.py +++ b/md_processing/v2/processors.py @@ -17,6 +17,10 @@ from md_processing.v2.extraction import DrECommand from md_processing.v2.parsing import AttributeFirstParser +from md_processing.v2.additional_properties import ( + AdditionalPropertiesUnreadable, merge_additional_properties, replace_note, stored_additional_properties, + unreadable_message, +) from md_processing.md_processing_utils.typed_properties import (_as_dict, classification_map, new_element_properties, not_applied_notes, parent_relationship_properties, @@ -1356,6 +1360,7 @@ async def _execute(self) -> Dict[str, Any]: f" | GUID={self.parsed_output.get('guid', 'new')} ══\033[0m" ) try: + await self._prepare_additional_properties() output = await self.apply_changes() finally: self._report_unapplied() @@ -1639,7 +1644,12 @@ async def fetch_element(self, guid: str, _max_timeout_retries: int = 2) -> Optio # Reordered subclients in EgeriaTech ensure this hits metadata-expert first res = await self.client._async_get_metadata_element_by_guid(guid) if res and isinstance(res, dict): - return res + # metadata-expert answers in the raw OpenMetadataElement shape (elementGUID, + # elementProperties.propertyValueMap); every caller reads the view-bean shape + # (elementHeader, properties), so convert it, or the guid, qualifiedName and + # Memento check silently read as absent. + from pyegeria.core.element_shape import as_view_element + return as_view_element(res) except Exception as e: logger.debug(f"MetadataExpert/Explorer fetch failed: {e}") @@ -2192,6 +2202,73 @@ async def sync_members(self, # Control flags rather than element properties: keep their defaults on an update. _KEEP_DEFAULTS_ON_UPDATE = {"Merge Update"} + #: True for a processor that writes additionalProperties keys of its own beside the document's + #: "Additional Properties" (ReportProcessor's report parameters), so an update always sends the map. + WRITES_ADDITIONAL_PROPERTIES = False + + def _is_merge_update(self) -> bool: + value = ((self.parsed_output.get("attributes") or {}).get("Merge Update") or {}).get("value", True) + return not (value is False or (isinstance(value, str) and value.strip().lower() == "false")) + + async def _prepare_additional_properties(self) -> None: + """Keep the stored additionalProperties keys the document doesn't write (md_processing/v2/additional_properties.py). + + Runs once, just before apply_changes(), for an update of an existing element (a Create that found + its element, or an Update). When the update sends additionalProperties -- the document gives + "Additional Properties", or the processor writes keys of its own -- the element's stored map is + read and the document's keys are laid over it, and the merged map becomes the "Additional + Properties" value every body builder reads. A replace update (Merge Update false) replaces, and + a warning names the stored keys it removes. A merge update whose stored map can't be read is + refused (AdditionalPropertiesUnreadable), never sent as a replacement. + """ + if not self.supports_target_element_lookup(): + return + is_update = bool(self.as_is_element) or ( + self.command.verb in ("Update", "Modify", "Upsert") + and bool(self.parsed_output.get("exists") or self.parsed_output.get("guid"))) + if not is_update: + return + attributes = self.parsed_output.get("attributes") + if attributes is None: + return + entry = attributes.get("Additional Properties") + document = entry.get("value") if isinstance(entry, dict) else None + if document in (None, "", {}): + document = None + elif not isinstance(document, dict): + return # not a parsed Dictionary: leave it to the processor + if document is None and not self.WRITES_ADDITIONAL_PROPERTIES: + return # nothing sent: Egeria keeps the stored map + + stored = stored_additional_properties(self.as_is_element) + if stored is None and self.as_is_element: + guid = (self.as_is_element.get("elementHeader") or {}).get("guid") + if guid: + try: + stored = stored_additional_properties(await self.fetch_element(guid)) + except Exception as e: # reported below as unreadable + logger.debug(f"_prepare_additional_properties: re-read of {guid} failed: {e}") + + if not self._is_merge_update(): + note = replace_note(stored, (document or {}).keys()) + if note: + self._add_warning(note) + return + if stored is None: + what = self.parsed_output.get("qualified_name") or self.parsed_output.get("guid") or "this element" + raise AdditionalPropertiesUnreadable(unreadable_message(f"'{what}'")) + + merged = merge_additional_properties(stored, document) + kept = sorted(set(stored) - set(document or {})) + if isinstance(entry, dict): + entry["document_value"] = document + entry["value"] = merged + else: + attributes["Additional Properties"] = {"value": merged, "style": "Dictionary", + "document_value": None, "kept_from_element": True} + if kept: + logger.info(f"Merge update keeps stored additionalProperties keys: {', '.join(kept)}") + def _drop_create_defaults(self) -> None: """Remove spec defaults the parser filled in for a Create that has become an Update. diff --git a/md_processing/v2/report.py b/md_processing/v2/report.py index f47fba4e..9cc4463d 100644 --- a/md_processing/v2/report.py +++ b/md_processing/v2/report.py @@ -38,6 +38,9 @@ from pyegeria.core._exceptions import PyegeriaNotFoundException +from md_processing.v2.additional_properties import ( + AdditionalPropertiesUnreadable, merge_additional_properties, stored_additional_properties, unreadable_message, +) from md_processing.v2.processors import AsyncBaseCommandProcessor from md_processing.md_processing_utils.common_md_utils import ( set_create_body, set_element_prop_body, update_element_dictionary, @@ -170,6 +173,11 @@ def _report_additional_properties(attributes: Dict[str, Any]) -> Dict[str, str]: user_extra = attributes.get("Additional Properties", {}).get("value") if isinstance(user_extra, dict): merged.update({str(k): str(v) for k, v in user_extra.items() if v is not None}) + from pyegeria.authoring.models import camel_copy_env + if not camel_copy_env(): + # D7 switch (PYEGERIA_REPORT_CAMEL_PARAMS=0): the parameters live only in the typed reportParameters + # document; additionalProperties keeps the author's own keys. + return merged # Arbitrary report-spec-specific params (e.g. {"collection_guid": "..."} # for the "Collection Members" report) that aren't part of the standard @@ -267,6 +275,12 @@ def _props_differ(desired: Dict[str, Any], current: Dict[str, Any]) -> bool: for key, value in desired.items(): if key in ("class", "typeName") or value is None: continue + if key == "additionalProperties" and isinstance(value, dict): + # merged key by key on update, so only the keys given count (additional_properties.py) + stored = current.get(key) or {} + if any(stored.get(k) != v for k, v in value.items()): + return True + continue if current.get(key) != value: return True return False @@ -288,6 +302,13 @@ async def upsert_dashboard_asset(client: Any, qualified_name: str, props: Dict[s if not _props_differ(props, existing.get("properties") or {}): return guid, "Unchanged", None update_props = filter_update(props, merge_update) if filter_update else props + stored = stored_additional_properties(existing) + if merge_update and update_props.get("additionalProperties") is not None: + # Egeria replaces the map whole: send the stored keys too (md_processing/v2/additional_properties.py). + if stored is None: + raise AdditionalPropertiesUnreadable(unreadable_message(f"Dashboard '{qualified_name}'")) + update_props = {**update_props, "additionalProperties": merge_additional_properties( + stored, update_props["additionalProperties"])} update_body = {"class": "UpdateElementRequestBody", "properties": update_props, "mergeUpdate": merge_update} await client._async_update_asset(guid, update_body) @@ -335,6 +356,10 @@ class ReportProcessor(AsyncBaseCommandProcessor): a spec with no element gets a note saying no link was written, not silence. """ + # The report parameters are additionalProperties keys, so every update sends the map: the base + # processor merges the stored keys in first (_prepare_additional_properties). + WRITES_ADDITIONAL_PROPERTIES = True + async def apply_changes(self) -> str: attributes = self.parsed_output["attributes"] qualified_name = self.parsed_output.get("qualified_name") or self.derive_qualified_name(attributes) diff --git a/pyegeria/authoring/__init__.py b/pyegeria/authoring/__init__.py new file mode 100644 index 00000000..d1d83557 --- /dev/null +++ b/pyegeria/authoring/__init__.py @@ -0,0 +1,48 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright Contributors to the ODPi Egeria project. +"""pyegeria.authoring: one typed authoring path for report specs, Reports and dashboard sheets (phase 1). + +Grown from `pyegeria.view.report_spec_publish` (Stage 1a). Dr.Egeria markdown stays the serialization: +the API writes and checks it, so authors don't have to. It never authenticates: the caller passes a +client signed in as the user, and a runner that runs documents as the user. + + draft = new_draft("report", display_name=..., spec=..., parameters={...}) + validate(draft, client) # structural, semantic, Dr.Egeria (with run=) + preview(draft, client=client) # a capped sample, run as the user; nothing registered + diff(old, new) # breaking changes flagged + plan = await async_publish_plan(draft, client, folders=[...]) # a new published version (D14) + await async_apply(plan, step="process", run=run, client=client, confirmed=True, digest=plan["digest"]) + +The undecided gates (D7, D14, D16, D4) are switches: see `AuthoringGates`. + +`AUTHORING_KEYS` (stable public path: `from pyegeria.authoring import AUTHORING_KEYS`) are the +additionalProperties keys this API writes for itself (publication stamp, draft marker, pin). They're +bookkeeping, never Report parameters: a consumer that passes a Report's additionalProperties to its spec +must skip them. +""" +from pyegeria.authoring.diff import diff +from pyegeria.authoring.drafts import ( + TEMPLATES, DraftStore, EgeriaDraftStore, LocalDraftStore, async_new_from, draft_store, new_draft, new_from, +) +from pyegeria.authoring.lifecycle import ( + KNOWN_CONSUMERS, apply, async_apply, async_history, async_load_held, async_publish_plan, history, load_held, + publish_plan, withdraw_plan, +) +from pyegeria.authoring.models import ( + AuthoringGates, AuthoringRefused, Change, Diff, Draft, ReportDraft, SheetDraft, SheetPlacement, SpecDraft, + ValidationResult, as_draft, default_gates, +) +from pyegeria.authoring.preview import async_preview, preview +from pyegeria.authoring.publication import AUTHORING_KEYS, STAMP_KEYS +from pyegeria.authoring.runner import inprocess_runner +from pyegeria.authoring.serialize import has_parent_id, report_parameter_maps, serialize +from pyegeria.authoring.validate import async_validate, validate + +__all__ = [ + "AUTHORING_KEYS", "STAMP_KEYS", "AuthoringGates", "AuthoringRefused", "Change", "Diff", "Draft", "DraftStore", "EgeriaDraftStore", + "KNOWN_CONSUMERS", "LocalDraftStore", "ReportDraft", "SheetDraft", "SheetPlacement", "SpecDraft", "TEMPLATES", + "ValidationResult", "apply", "as_draft", "async_apply", "async_publish_plan", "async_history", "async_load_held", "async_new_from", "async_preview", + "async_validate", "default_gates", "diff", "draft_store", "has_parent_id", "history", "inprocess_runner", + "load_held", "new_draft", "new_from", "preview", "publish_plan", "report_parameter_maps", "serialize", + "validate", "withdraw_plan", +] diff --git a/pyegeria/authoring/diff.py b/pyegeria/authoring/diff.py new file mode 100644 index 00000000..b6755f85 --- /dev/null +++ b/pyegeria/authoring/diff.py @@ -0,0 +1,113 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright Contributors to the ODPi Egeria project. +"""diff(a, b): a structured comparison of two drafts of the same kind. + +Areas: field, column, parameter, analytic parameter, placement. **Breaking** (needs confirming before a +promote, design note §1d): a removed column or placement, a changed parameter type, a changed target type. +A changed analytic routine is flagged breaking too: everything that reads the spec's columns depends on it. + +A removed placement and a placement reorder are reported, but publishing doesn't apply them to a sheet that +already exists (Dr.Egeria's sheet commands add and update only); their `why` says so. +""" +from __future__ import annotations + +from typing import Any, Dict, List, Tuple + +from pyegeria.authoring.models import AuthoringRefused, Change, Diff, ReportDraft, SheetDraft, SpecDraft, as_draft + + +NOT_PUBLISHED_REMOVE = ("publishing an update doesn't remove it: the placement stays on the existing sheet " + "(Dr.Egeria's sheet commands add and update only)") +NOT_PUBLISHED_ORDER = ("publishing an update doesn't reorder an existing sheet: placements keep their positions " + "(Dr.Egeria's sheet commands add and update only)") + + +def _fields(diff: Diff, a: Any, b: Any, names: List[str], breaking: Tuple[str, ...] = ()) -> None: + for name in names: + before, after = getattr(a, name, None), getattr(b, name, None) + if before != after: + diff.changes.append(Change(area="field", name=name, change="changed", before=before, after=after, + breaking=name in breaking, + why="the elements it runs over change" if name in breaking else None)) + + +def _columns(fs) -> Dict[str, Dict[str, Any]]: + """{key: {"name", "formats": [...], ...}} across every format row.""" + out: Dict[str, Dict[str, Any]] = {} + for fmt in fs.formats or []: + types = list(getattr(fmt, "types", None) or []) + for col in getattr(fmt, "attributes", None) or []: + data = col if isinstance(col, dict) else col.model_dump(exclude_none=True) + key = data.get("key") + if not key: + continue + entry = out.setdefault(key, {**{k: v for k, v in data.items() if k != "key"}, "formats": []}) + entry["formats"] = sorted(set(entry["formats"]) | set(types)) + return out + + +def _keyed(diff: Diff, area: str, before: Dict[str, Any], after: Dict[str, Any], *, removed_breaks: bool, + type_breaks: bool = False) -> None: + for key in sorted(set(before) | set(after)): + if key not in after: + diff.changes.append(Change(area=area, name=key, change="removed", before=before[key], + breaking=removed_breaks, + why=f"anything that reads this {area} loses it" if removed_breaks else None)) + elif key not in before: + diff.changes.append(Change(area=area, name=key, change="added", after=after[key])) + elif before[key] != after[key]: + typed = type_breaks and type(before[key]) is not type(after[key]) + diff.changes.append(Change(area=area, name=key, change="changed", before=before[key], after=after[key], + breaking=typed, + why=f"its type changed ({type(before[key]).__name__} -> " + f"{type(after[key]).__name__})" if typed else None)) + + +def _action(fs) -> Dict[str, Any]: + action = fs.action + if action is None: + return {} + return action if isinstance(action, dict) else action.model_dump(exclude_none=True) + + +def diff(a: Any, b: Any) -> Diff: + a, b = as_draft(a), as_draft(b) + if a.kind != b.kind: + raise AuthoringRefused(f"Two drafts of the same kind are compared, not a {a.kind} and a {b.kind}.") + d = Diff() + if isinstance(a, SpecDraft): + _fields(d, a, b, ["label", "qualified_name"]) + _fields(d, a.format_set, b.format_set, ["target_type", "heading", "description", "family", "aliases"], + breaking=("target_type",)) + _keyed(d, "column", _columns(a.format_set), _columns(b.format_set), removed_breaks=True) + act_a, act_b = _action(a.format_set), _action(b.format_set) + for name in ("function", "analytic_function"): + if act_a.get(name) != act_b.get(name): + routine = name == "analytic_function" + d.changes.append(Change(area="field", name=f"action.{name}", change="changed", before=act_a.get(name), + after=act_b.get(name), breaking=routine, + why="a different routine produces different columns" if routine else None)) + params_a = {p: "required" for p in act_a.get("required_params") or []} + params_a.update({p: "optional" for p in act_a.get("optional_params") or []}) + params_b = {p: "required" for p in act_b.get("required_params") or []} + params_b.update({p: "optional" for p in act_b.get("optional_params") or []}) + _keyed(d, "parameter", params_a, params_b, removed_breaks=False) + return d + if isinstance(a, ReportDraft): + _fields(d, a, b, ["display_name", "qualified_name", "description", "spec", "output_format", "pinned_version"]) + _keyed(d, "parameter", a.parameters, b.parameters, removed_breaks=False, type_breaks=True) + _keyed(d, "analytic parameter", a.analytic_params, b.analytic_params, removed_breaks=False, type_breaks=True) + return d + _fields(d, a, b, ["name", "heading", "description", "family", "perspectives"]) + pa = {p.name: p.model_dump(exclude={"name"}) for p in a.placements} + pb = {p.name: p.model_dump(exclude={"name"}) for p in b.placements} + _keyed(d, "placement", pa, pb, removed_breaks=True) + for c in d.changes: + if c.area == "placement" and c.change == "removed": + c.why = f"{c.why}; but {NOT_PUBLISHED_REMOVE}" if c.why else NOT_PUBLISHED_REMOVE + order_a = [p.name for p in a.placements if p.name in pb] + order_b = [p.name for p in b.placements if p.name in pa] + if order_a != order_b: + d.changes.append(Change(area="placement", name="(order)", change="changed", before=order_a, after=order_b, + why=NOT_PUBLISHED_ORDER)) + return d diff --git a/pyegeria/authoring/drafts.py b/pyegeria/authoring/drafts.py new file mode 100644 index 00000000..d3d54c71 --- /dev/null +++ b/pyegeria/authoring/drafts.py @@ -0,0 +1,508 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright Contributors to the ODPi Egeria project. +"""Making drafts, and keeping them (D16). + +- `new_draft(kind, **fields)`: a new draft of a kind. +- `new_from(prototype=(kind, label) | template=name, new_label=...)`: a copy of an existing artifact (or a + template), renamed, with `derived_from` recorded. A spec prototype is read from pyegeria's registry (read + only, nothing registered); a held Report or sheet is read from Egeria as the caller (`client`). +- `DraftStore`: where drafts are kept (D16, decided: Egeria, private, 30-day prune). `EgeriaDraftStore`, the + default for apps, keeps the draft as the element itself in the author's private zone; `LocalDraftStore` + (a directory per user, given by the caller) is for tests and offline use. +""" +from __future__ import annotations + +import copy +import json +import os +import re +from pathlib import Path +from typing import Any, Dict, List, Optional, Protocol, Tuple, Union + +from pyegeria.authoring.publication import stamp_of +from pyegeria.authoring.models import ( + KINDS, AuthoringRefused, ReportDraft, SheetDraft, SheetPlacement, SpecDraft, as_draft, draft_key, +) +from pyegeria.view._output_format_models import Column, Format, FormatSet + +AnyDraft = Union[SpecDraft, ReportDraft, SheetDraft] + + +def new_draft(kind: str, **fields: Any) -> AnyDraft: + """A new draft of `kind` ("spec", "report" or "sheet") from its fields. A spec takes `label` and + `format_set` (a FormatSet or its dict).""" + if kind not in KINDS: + raise AuthoringRefused(f"A draft is one of {', '.join(KINDS)}, not {kind!r}.") + return as_draft({"kind": kind, **fields}) + + +# ── Templates (the built-in tier: blanks with guidance) ────────────────────── + +def _template_spec(label: str) -> SpecDraft: + fs = FormatSet(target_type="Referenceable", heading=label, description="Describe what this report answers.", + formats=[Format(types=["ALL"], attributes=[ + Column(name="Display Name", key="display_name"), + Column(name="Qualified Name", key="qualified_name"), + Column(name="Description", key="description")])], + action={"function": "ClassificationExplorer.find_elements_by_property_value", + "required_params": ["search_string"], "optional_params": ["page_size"], + "spec_params": {}}) + return SpecDraft(label=label, format_set=fs) + + +TEMPLATES: Dict[str, Tuple[str, Any]] = { + "find-spec": ("spec", _template_spec), + "report": ("report", lambda label: ReportDraft(display_name=label, spec="Referenceable", output_format="TABLE", + parameters={"search_string": "*", "page_size": 25})), + "sheet": ("sheet", lambda label: SheetDraft(name=label, heading=label, placements=[ + SheetPlacement(text="**What this sheet shows**\n\nDescribe the sheet here.", name="About this sheet", + span="full", emphasis="panel")])), +} + + +def _rename(draft: AnyDraft, new_label: str) -> AnyDraft: + if isinstance(draft, SpecDraft): + fs = draft.format_set.model_copy(deep=True) + fs.aliases = [] # an alias of the prototype would make two specs answer to one name + return SpecDraft(label=new_label, format_set=fs) + if isinstance(draft, ReportDraft): + data = draft.model_dump(exclude={"qualified_name", "guid", "base_version", "derived_from"}) + return ReportDraft(**{**data, "display_name": new_label}) + data = draft.model_dump(exclude={"guid", "base_version", "derived_from"}) + return SheetDraft(**{**data, "name": new_label, "heading": new_label}) + + +def _spec_from_registry(label: str) -> SpecDraft: + from pyegeria.view.base_report_formats import lookup_report_spec + fs, _tier, found = lookup_report_spec(label) + if fs is None: + raise AuthoringRefused(f"No report spec named {label!r} is known here to copy (pyegeria's registry, " + f"its Egeria-held and local tiers).") + return SpecDraft(label=found or label, format_set=fs.model_copy(deep=True)) + + +async def async_new_from(*, prototype: Optional[Tuple[str, str]] = None, template: Optional[str] = None, + new_label: str, client: Any = None) -> AnyDraft: + """A copy of a prototype `(kind, label)` or a template, named `new_label`, with `derived_from` recorded. + A held Report or sheet prototype is read from Egeria as the caller (`client`).""" + if (prototype is None) == (template is None): + raise AuthoringRefused("Give one of prototype=(kind, label) or template=name.") + if not (new_label or "").strip(): + raise AuthoringRefused("A new draft needs a new name (new_label).") + if template is not None: + if template not in TEMPLATES: + raise AuthoringRefused(f"No template {template!r}; the templates are {', '.join(sorted(TEMPLATES))}.") + draft = TEMPLATES[template][1](new_label) + draft.derived_from = f"template:{template}" + return draft + kind, label = prototype + if kind == "spec": + source = _spec_from_registry(label) + elif kind in ("report", "sheet"): + if client is None: + raise AuthoringRefused(f"Copying a held {kind} reads it from Egeria as you: a client is needed.") + from pyegeria.authoring.lifecycle import async_load_held + source = await async_load_held(kind, label, client) + else: + raise AuthoringRefused(f"A prototype is one of {', '.join(KINDS)}, not {kind!r}.") + if draft_key(source) == new_label: + raise AuthoringRefused(f"The copy needs a name of its own: {new_label!r} is the prototype's.") + draft = _rename(source, new_label) + draft.derived_from = f"prototype:{kind}:{label}" + return draft + + +def new_from(*, prototype: Optional[Tuple[str, str]] = None, template: Optional[str] = None, new_label: str, + client: Any = None) -> AnyDraft: + from pyegeria.core.async_runner import run_sync + return run_sync(async_new_from(prototype=prototype, template=template, new_label=new_label, client=client)) + + +# ── Draft storage (D16) ─────────────────────────────────────────────────────── + +class DraftStore(Protocol): + """Where a user's drafts are kept. Every call names the user: a draft is personal (§1b tier 1).""" + def save(self, user_id: str, draft: AnyDraft) -> str: ... + def load(self, user_id: str, kind: str, name: str) -> Optional[AnyDraft]: ... + def list(self, user_id: str, kind: Optional[str] = None) -> List[Dict[str, str]]: ... + def delete(self, user_id: str, kind: str, name: str) -> bool: ... + + +_USER_OK = re.compile(r"[A-Za-z0-9][A-Za-z0-9._@-]{0,127}") + + +def _slug(name: str) -> str: + slug = re.sub(r"[^A-Za-z0-9._-]+", "-", name).strip("-.")[:120] + if not slug: + raise AuthoringRefused(f"Can't make a file name from {name!r}.") + return slug + + +class LocalDraftStore: + """Drafts as JSON files: `///.json`. The caller chooses `root` (the API never + picks a shared default) and passes the user id it signed in; a user id that isn't a plain name is + refused, so one user can't reach another's directory.""" + + def __init__(self, root: Union[str, Path]): + self.root = Path(root).expanduser() + + def _dir(self, user_id: str, kind: str) -> Path: + if not _USER_OK.fullmatch(user_id or ""): + raise AuthoringRefused(f"Not a user id this store accepts: {user_id!r}.") + if kind not in KINDS: + raise AuthoringRefused(f"A draft is one of {', '.join(KINDS)}, not {kind!r}.") + return self.root / user_id / kind + + def _path(self, user_id: str, kind: str, name: str) -> Path: + return self._dir(user_id, kind) / f"{_slug(name)}.json" + + def save(self, user_id: str, draft: AnyDraft) -> str: + draft = as_draft(draft) + path = self._path(user_id, draft.kind, draft_key(draft)) + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_suffix(".json.tmp") + tmp.write_text(json.dumps({"name": draft_key(draft), "draft": draft.model_dump(mode="json")}, + indent=2, sort_keys=True)) + os.replace(tmp, path) + return str(path) + + def load(self, user_id: str, kind: str, name: str) -> Optional[AnyDraft]: + path = self._path(user_id, kind, name) + if not path.is_file(): + return None + return as_draft(json.loads(path.read_text())["draft"]) + + def list(self, user_id: str, kind: Optional[str] = None) -> List[Dict[str, str]]: + out = [] + for k in ([kind] if kind else list(KINDS)): + d = self._dir(user_id, k) + for f in sorted(d.glob("*.json")) if d.is_dir() else []: + try: + out.append({"kind": k, "name": json.loads(f.read_text())["name"]}) + except (OSError, ValueError, KeyError): + out.append({"kind": k, "name": f.stem, "note": "unreadable draft file"}) + return out + + def delete(self, user_id: str, kind: str, name: str) -> bool: + path = self._path(user_id, kind, name) + if path.is_file(): + path.unlink() + return True + return False + + +_ELEMENT_TYPE = {"spec": "ReportSpec", "report": "Report", "sheet": "Dashboard"} +PRUNE_DAYS = 30 +#: The draft marker on a Report (additionalProperties); a spec carries contentStatus DRAFT instead. +DRAFT_MARKER = ("authoringState", "draft") + + +def _parse_time(value: Any): + from datetime import datetime, timezone + if value in (None, ""): + return None + if isinstance(value, (int, float)): + return datetime.fromtimestamp(value / 1000 if value > 1e11 else value, tz=timezone.utc) + text = str(value).replace("Z", "+00:00") + try: + parsed = datetime.fromisoformat(text) + except ValueError: + return None + return parsed if parsed.tzinfo else parsed.replace(tzinfo=timezone.utc) + + +def _is_marked(kind: str, el: Dict[str, Any]) -> bool: + """Whether an element carries the draft marker. A sheet has no marker Dr.Egeria writes yet, so it is never + "marked": prune checks a sheet separately (`EgeriaDraftStore._sheet_prunable`) and keeps it unless it can + show the sheet is an unfiled private draft.""" + props = el.get("properties") or {} + if kind == "spec": + return str(props.get("contentStatus") or "").upper() == "DRAFT" + if kind == "report": + return (props.get("additionalProperties") or {}).get(DRAFT_MARKER[0]) == DRAFT_MARKER[1] + return False + + +def _is_not_found(exc: BaseException) -> bool: + """A lookup that failed because nothing matched (as opposed to a timeout, a permission refusal, ...).""" + from pyegeria.core._exceptions import PyegeriaNotFoundException + return isinstance(exc, PyegeriaNotFoundException) + + +class EgeriaDraftStore: + """Drafts held in Egeria, zoned private to their author (D16, decided "Egeria, private, 30-day prune"). + + - **The draft is the element itself** (a ReportSpec, a Report, a Dashboard sheet chain) with its real + qualified name, zoned `[]` and nothing else (ANSWER-DRAFT-PRIVATE-ZONE-2026-11-13, + code-verified on the build laz runs): Egeria's validateZoneAccess grants a zone named for a user to that + user only, and the search pushdown leaves other users' drafts out of their searches. No marker zone: a + control defined for it later would expose every user's drafts. The anchor is zoned (the Dashboard, + ReportSpec or Report); its children inherit through Anchors. + - **Marked as a draft on the element:** a spec with contentStatus DRAFT, a Report with additionalProperties + `authoringState: draft`. A sheet has no carrier Dr.Egeria writes yet (Create Dashboard Sheet sets the + Dashboard's own additionalProperties), so a sheet draft is known by its zone alone. + - **A userId that names a governance zone is refused** (that user would read the zone's elements), as is + one in `controlled_zones` (zones the app knows carry a security control). + - **Written as the author:** every write is a Dr.Egeria document run through the caller's runner (`run`), + and the client must be signed in as the user whose draft it is. + - **Never over a published element:** saving a draft whose element is already held outside the author's + private zone is refused (it would change what others see); make a copy with new_from, or promote. + - **Never filed into a consumer folder:** a saved draft is only its element. Promotion files it. + - **Promotion clears the zone:** promotion_plan() is the publish plan with Declassify Zone Membership (a + blank Zone Membership would keep the zone), marking the element ACTIVE / published. After promoting a + sheet, check that another user's search finds its placements, not only the sheet (anchor zones). + - **prune_drafts(older_than_days=30)** removes the author's unpromoted drafts not updated for more than 30 + days, as the author. It can't see, and never sweeps, anyone else's drafts. It removes only an element + whose zones are exactly `[]` (a zone list that merely includes the userId is shared). A spec or + Report must also carry the draft marker. A sheet has no marker, so it must also have a `Dashboard::` + qualified name and belong to no collection (a sheet filed in a folder is published, even if its + Declassify step didn't run); when its memberships can't be read it is kept. `removed` lists only what + was actually deleted. + """ + + def __init__(self, client: Any, run: Any = None, *, controlled_zones: Any = ()): + self.client = client + self.run = run + self.controlled_zones = set(controlled_zones or ()) + self._zone_checked: Dict[str, bool] = {} + + # ---------------------------------------------------------------- helpers + + @staticmethod + def zone(user_id: str) -> str: + """The author's private zone: their user id.""" + return user_id + + async def _check_zone_name(self, user_id: str) -> None: + """Refuse a user id that is also a governance zone's name (or one the app says is controlled).""" + if self._zone_checked.get(user_id): + return + if user_id in self.controlled_zones: + raise AuthoringRefused(f"User id {user_id!r} is also the name of a controlled zone, so a draft zoned " + f"[{user_id}] wouldn't be private. Drafts can't be kept in Egeria for this user.") + try: + found = await self.client.classification_manager._async_get_elements_by_property_value( + user_id, ["zoneName", "qualifiedName"], metadata_element_type="GovernanceZone", starts_with=False, + ends_with=False, ignore_case=True, page_size=10, graph_query_depth=0) + except Exception as exc: # noqa: BLE001 + if not _is_not_found(exc): # a "not found" answer is an exception on some paths; anything else + raise AuthoringRefused( # means the check wasn't done, so the draft isn't saved (fail closed) + f"Couldn't check whether user id {user_id!r} is also a governance zone's name " + f"({type(exc).__name__}), so a draft zoned [{user_id}] can't be shown to be private. " + f"Nothing was saved; try again.") from exc + found = [] + names = {str((e.get("properties") or {}).get("zoneName") or "").lower() for e in found + if isinstance(e, dict)} if isinstance(found, list) else set() + if user_id.lower() in names: + raise AuthoringRefused(f"User id {user_id!r} is also the name of a governance zone, so a draft zoned " + f"[{user_id}] wouldn't be private. Drafts can't be kept in Egeria for this user.") + self._zone_checked[user_id] = True + + def _author(self, user_id: str) -> str: + signed_in = getattr(self.client, "user_id", None) + if not user_id or user_id != signed_in: + raise AuthoringRefused(f"Drafts are written and read as their author: this client is signed in as " + f"{signed_in!r}, not {user_id!r}.") + return self.zone(user_id) + + async def _element(self, kind: str, name: str) -> Optional[Dict[str, Any]]: + from pyegeria.authoring.lifecycle import _by_qualified_name, _report_element + if kind == "spec": + from md_processing.v2.report import find_report_spec_element + return await find_report_spec_element(self.client, name) + if kind == "report": + return await _report_element(self.client, name) + return await _by_qualified_name(self.client, f"Dashboard::{name}") + + @staticmethod + def _zones(el: Dict[str, Any]) -> List[str]: + from pyegeria.view.egeria_spec_cache import _zones + return _zones(el) + + # ---------------------------------------------------------------- the store + + async def async_save(self, user_id: str, draft: AnyDraft) -> str: + """Create or update the draft's element in the author's private zone. Returns its qualified name.""" + from pyegeria.authoring.serialize import serialize + from pyegeria.view.report_spec_publish import _call + zone = self._author(user_id) + draft = as_draft(draft) + if self.run is None: + raise AuthoringRefused("Saving a draft to Egeria runs a document as you: a runner is needed.") + await self._check_zone_name(user_id) + held = await self._element(draft.kind, draft_key(draft)) + if held is not None and zone not in self._zones(held): + raise AuthoringRefused(f"{draft.kind} {draft_key(draft)!r} is already held in Egeria outside your private " + f"draft zone ({', '.join(self._zones(held)) or 'public'}): saving the draft there " + f"would change what others see. Copy it with new_from, or promote the draft.") + env = _call(self.run, serialize(draft, zones=[zone], as_draft_element=True), "process") + if not env.get("success"): + words = "; ".join(str(e.get("message", e)) if isinstance(e, dict) else str(e) + for e in (env.get("execution_errors") or env.get("validation_errors") or [])[:3]) + raise AuthoringRefused(f"The draft was not saved: {words or 'Dr.Egeria did not succeed'}.") + if isinstance(draft, SpecDraft): + return draft.qualified_name + if isinstance(draft, SheetDraft): + return f"Dashboard::{draft.name}" + return draft.qualified_name or draft.display_name + + async def async_load(self, user_id: str, kind: str, name: str) -> Optional[AnyDraft]: + """The author's draft `name`, or None when no element by that name is in their private zone.""" + from pyegeria.authoring.lifecycle import async_load_held + zone = self._author(user_id) + el = await self._element(kind, name) + if el is None or zone not in self._zones(el): + return None + return await async_load_held(kind, name, self.client) + + async def async_list(self, user_id: str, kind: Optional[str] = None) -> List[Dict[str, Any]]: + """The author's drafts: every ReportSpec, Report and Dashboard in their private zone, read as them.""" + zone = self._author(user_id) + out: List[Dict[str, Any]] = [] + for k in ([kind] if kind else list(KINDS)): + start = 0 + while True: + page = await self.client.classification_manager._async_find_elements_by_property_value( + ".*", ["qualifiedName"], metadata_element_type=_ELEMENT_TYPE[k], starts_with=False, + governance_zone_filter=[zone], graph_query_depth=0, start_from=start, page_size=50) + raw = [e for e in page if isinstance(e, dict) and e.get("elementHeader")] if isinstance(page, list) else [] + if not raw: # only an empty page ends the read (a short page isn't the last page) + break + start += len(raw) + for el in (e for e in raw if zone in self._zones(e)): + props = el.get("properties") or {} + versions = (el.get("elementHeader") or {}).get("versions") or {} + name = props.get("displayName") or props.get("qualifiedName") + if k == "sheet" and str(props.get("qualifiedName", "")).startswith("Dashboard::"): + name = props["qualifiedName"].split("::", 1)[1] + out.append({"kind": k, "name": name, "qualified_name": props.get("qualifiedName"), + "guid": el["elementHeader"].get("guid"), + "created": versions.get("createTime"), "updated": versions.get("updateTime"), + "zones": self._zones(el), "marked": _is_marked(k, el), + "published_version": stamp_of(el)}) + return out + + async def async_delete(self, user_id: str, kind: str, name: str) -> bool: + """Delete the author's draft (only an element in their private zone), as the author.""" + zone = self._author(user_id) + el = await self._element(kind, name) + if el is None or zone not in self._zones(el): + return False + if stamp_of(el): + raise AuthoringRefused(f"{kind} {name!r} is published (version {stamp_of(el)}): a published element is " + f"never deleted (D14), so Reports pinned to it keep reading it.") + guid = el["elementHeader"]["guid"] + if kind == "sheet": + from md_processing.v2.dashboard_sheet import DashboardSheetEgeriaWriter + return bool(await DashboardSheetEgeriaWriter(self.client).delete_sheet(name)) + if kind == "report": + await self.client.asset_maker._async_delete_asset(guid) + else: + await self.client.metadata_expert._async_delete_metadata_element(guid) + return True + + async def _sheet_prunable(self, d: Dict[str, Any]) -> Optional[str]: + """None when the sheet `d` (from async_list) may be pruned, else the reason it is kept.""" + if not str(d.get("qualified_name") or "").startswith("Dashboard::"): + return "not a Dashboard:: sheet" + try: + related = await self.client.classification_manager._async_get_related_elements( + d["guid"], "CollectionMembership", start_at_end=2, page_size=0, graph_query_depth=0) + except Exception as exc: # noqa: BLE001 + if not _is_not_found(exc): + return f"its folder memberships couldn't be read ({type(exc).__name__})" + related = [] + if isinstance(related, list) and any(isinstance(r, dict) for r in related): + return "filed in a collection (published)" + if not isinstance(related, (list, str)): + return "its folder memberships couldn't be read" + return None + + async def async_prune_drafts(self, user_id: str, older_than_days: int = PRUNE_DAYS, *, + dry_run: bool = False) -> Dict[str, Any]: + """Remove the author's drafts not updated for more than `older_than_days` (a promoted draft is no longer + in the private zone, so it's never pruned). Run as the author only. See the class doc for what counts + as a draft; `kept_reasons` says why each in-zone element older than the cutoff was kept.""" + from datetime import datetime, timedelta, timezone + cutoff = datetime.now(timezone.utc) - timedelta(days=older_than_days) + zone = self._author(user_id) + drafts = await self.async_list(user_id) + out: Dict[str, Any] = {"removed": [], "kept": [], "not_dated": [], "not_marked": [], "not_deleted": [], + "kept_reasons": {}, "dry_run": dry_run, "older_than_days": older_than_days} + for d in drafts: + key = f"{d['kind']}:{d['name']}" + when = _parse_time(d.get("updated")) or _parse_time(d.get("created")) + if when is None: + out["not_dated"].append(d["name"]) # never removed on a guess + continue + if when >= cutoff: + out["kept"].append(key) + continue + if d.get("published_version"): + reason = f"published (version {d['published_version']}): a published element is never deleted" + elif list(d.get("zones") or []) != [zone]: + reason = "zoned beyond your private zone (shared)" + elif d["kind"] == "sheet": + reason = await self._sheet_prunable(d) + elif not d.get("marked"): + out["not_marked"].append(d["name"]) # in the zone without the draft marker: never removed + continue + else: + reason = None + if reason: + out["kept_reasons"][key] = reason + continue + if dry_run: + out["removed"].append(key) # would be removed + elif await self.async_delete(user_id, d["kind"], d["name"]): + out["removed"].append(key) + else: + out["not_deleted"].append(key) # the delete found nothing to delete, or refused + return out + + async def async_promotion_plan(self, user_id: str, kind: str, name: str, *, folders: Any = ()) -> Dict[str, Any]: + """The publish plan for the author's draft: it clears the private zone and files it in `folders`.""" + from pyegeria.authoring.lifecycle import async_publish_plan + self._author(user_id) + draft = await self.async_load(user_id, kind, name) + if draft is None: + raise AuthoringRefused(f"You have no {kind} draft named {name!r} in your private zone.") + # The target is resolved as the author (its private zone comes back in `held`, so the plan stamps the + # element, then declassifies it). + return await async_publish_plan(draft, self.client, folders=folders) + + # ---------------------------------------------------------------- sync forms (DraftStore) + + def _sync(self, coro): + from pyegeria.core.async_runner import run_sync + return run_sync(coro) + + def save(self, user_id: str, draft: AnyDraft) -> str: + return self._sync(self.async_save(user_id, draft)) + + def load(self, user_id: str, kind: str, name: str) -> Optional[AnyDraft]: + return self._sync(self.async_load(user_id, kind, name)) + + def list(self, user_id: str, kind: Optional[str] = None) -> List[Dict[str, Any]]: + return self._sync(self.async_list(user_id, kind)) + + def delete(self, user_id: str, kind: str, name: str) -> bool: + return self._sync(self.async_delete(user_id, kind, name)) + + def prune_drafts(self, user_id: str, older_than_days: int = PRUNE_DAYS, *, dry_run: bool = False) -> Dict[str, Any]: + return self._sync(self.async_prune_drafts(user_id, older_than_days, dry_run=dry_run)) + + def promotion_plan(self, user_id: str, kind: str, name: str, *, folders: Any = ()) -> Dict[str, Any]: + return self._sync(self.async_promotion_plan(user_id, kind, name, folders=folders)) + + +def draft_store(backend: str = "egeria", **kwargs: Any) -> DraftStore: + """The store for a backend: "egeria" (the default for apps: client=, run=) or "local" (root=).""" + if backend == "local": + if not kwargs.get("root"): + raise AuthoringRefused("The local draft store needs a directory (root) from the caller.") + return LocalDraftStore(kwargs["root"]) + if backend == "egeria": + return EgeriaDraftStore(kwargs.get("client"), kwargs.get("run"), + controlled_zones=kwargs.get("controlled_zones", ())) + raise AuthoringRefused(f"No draft store {backend!r}; there are 'egeria' and 'local'.") diff --git a/pyegeria/authoring/lifecycle.py b/pyegeria/authoring/lifecycle.py new file mode 100644 index 00000000..997121fb --- /dev/null +++ b/pyegeria/authoring/lifecycle.py @@ -0,0 +1,437 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright Contributors to the ODPi Egeria project. +"""Held artifacts: load one as a draft, read its history, plan its publish or withdraw, apply a plan. + +Every Egeria read here uses the caller's `client`, signed in as the user; every write runs through the +caller's runner (`run(markdown, directive)`), as the user. This module never authenticates and never +uses a service account. +""" +from __future__ import annotations + +import json +from typing import Any, Dict, Iterable, List, Optional, Union + +from loguru import logger + +from pyegeria.authoring.models import ( + AuthoringGates, AuthoringRefused, ReportDraft, SheetDraft, SheetPlacement, SpecDraft, as_draft, default_gates, +) +from pyegeria.authoring.publication import AUTHORING_KEYS +from pyegeria.authoring.serialize import ( + DASHBOARD_FOLDER_PREFIX, FIND_PARAMETERS, PINNED_VERSION_KEY, SPEC_FOLDER_PREFIX, serialize, +) +from pyegeria.view.report_spec_publish import ( + PublishRefused, build_publish_document, build_withdraw_document, document_digest, run_document, section, +) + +#: The consumers that have folders today (design note §1). A folder is ``. +KNOWN_CONSUMERS = ("EgeriaAdvisor", "Jupyter", "MyEgeria", "Portal", "ResourceExplorer", "pyegeria") +RESTRICTED_LATER = "Restricted publish comes later, after the zone proof: publish as public for now." +PIN_REFUSED = "Pinning a spec version is switched off here (AuthoringGates.allow_pinning, D14)." +#: A pin is recorded (reportSpecVersion) and honoured by authoring validate/preview, but nothing that runs a +#: placed Report reads it yet (read-back 2026-11-13 F3), so the live sheet runs the latest version. +PIN_NOT_HONOURED = ("the pin to version {version} is recorded, but dashboards don't read it yet: a placement of " + "this Report runs the spec's latest version until the spec resolver reads reportSpecVersion") +#: Dr.Egeria's sheet commands add and update placements only (read-back 2026-11-13 F4). +SHEET_UPDATE_LIMITS = ("if the sheet already exists, placements are added or updated in place: a placement no " + "longer in this draft stays on the sheet, and existing placements keep their positions " + "(removing or reordering placements isn't something publishing can do yet)") +BUILTIN_REFUSED = ("'{label}' is a built-in spec: it ships with pyegeria and isn't published from here. " + "Copy it with new_from(prototype=('spec', '{label}'), new_label=...) and publish the copy.") + + +# ── Reading held artifacts, as the caller ───────────────────────────────────── + +async def _by_qualified_name(client: Any, qn: str) -> Optional[Dict[str, Any]]: + from md_processing.v2.report import find_element_by_qualified_name + return await find_element_by_qualified_name(client, qn) + + +async def _report_element(client: Any, label: str) -> Optional[Dict[str, Any]]: + """A Report by qualified name, else by exact display name (one Report must match).""" + el = await _by_qualified_name(client, label) + if el and _type_name(el) == "Report": + return el + found = await client.classification_manager._async_get_elements_by_property_value( + label, ["displayName"], metadata_element_type="Report", starts_with=False, ends_with=False, + ignore_case=False, page_size=25, graph_query_depth=0) + matches = [e for e in found if isinstance(e, dict) and (e.get("properties") or {}).get("displayName") == label] \ + if isinstance(found, list) else [] + if len(matches) > 1: + raise AuthoringRefused(f"{len(matches)} Reports are named {label!r}; give its qualified name.") + return matches[0] if matches else None + + +def _type_name(el: Dict[str, Any]) -> str: + return ((el.get("elementHeader") or {}).get("type") or {}).get("typeName", "") + + +def _version(el: Dict[str, Any]) -> Optional[int]: + v = ((el.get("elementHeader") or {}).get("versions") or {}).get("version") + try: + return int(v) + except (TypeError, ValueError): + return None + + +def report_draft_from_element(el: Dict[str, Any]) -> ReportDraft: + """A ReportDraft from a Report element. Parameters are read from the typed `reportParameters` first + (D7), then from the camelCase additionalProperties keys; `notes` says which was read.""" + from md_processing.v2.report import _camel_to_snake + props = el.get("properties") or {} + extra = dict(props.get("additionalProperties") or {}) + notes: List[str] = [] + analytic: Dict[str, Any] = {} + raw_analytic = extra.get("analyticParams") + if raw_analytic: + try: + analytic = json.loads(raw_analytic) if isinstance(raw_analytic, str) else dict(raw_analytic) + except (TypeError, ValueError): + notes.append("analyticParams is not JSON; not read") + params: Dict[str, Any] = {} + pinned = extra.pop(PINNED_VERSION_KEY, None) + try: + pinned = int(pinned) if pinned not in (None, "") else None + except (TypeError, ValueError): + notes.append(f"{PINNED_VERSION_KEY} {pinned!r} is not a version number; following the latest version") + pinned = None + typed = props.get("reportParameters") + if typed: + try: + doc = json.loads(typed) if isinstance(typed, str) else dict(typed) + params = {k: v for k, v in doc.items() if k not in analytic} + notes.append("parameters read from reportParameters") + except (TypeError, ValueError): + notes.append("reportParameters is not JSON; read the additionalProperties keys instead") + typed = None + if not typed: + for key, value in extra.items(): + if key in ("reportSpec", "outputFormat", "analyticParams"): + continue + if key in AUTHORING_KEYS: + continue # the authoring API's own bookkeeping (stamp, marker), never a parameter + params[_camel_to_snake(key)] = _decode(value) if _camel_to_snake(key) in FIND_PARAMETERS else value + notes.append("parameters read from the additionalProperties keys (no reportParameters)") + return ReportDraft(display_name=props.get("displayName") or "", qualified_name=props.get("qualifiedName"), + description=props.get("description"), spec=extra.get("reportSpec") or "", + output_format=props.get("outputFormat") or extra.get("outputFormat") or "DICT", + parameters=params, analytic_params=analytic, pinned_version=pinned, + guid=(el.get("elementHeader") or {}).get("guid"), + base_version=_version(el), notes=notes) + + +def _decode(value: Any) -> Any: + """A camelCase-copy value back to its type: "True" -> True, "25" -> 25, "['A']" / '["A"]' -> ["A"].""" + if not isinstance(value, str): + return value + text = value.strip() + if text in ("True", "False"): + return text == "True" + if text.lstrip("-").isdigit(): + return int(text) + if text.startswith("[") and text.endswith("]"): + try: + return json.loads(text) + except ValueError: + import ast + try: + return list(ast.literal_eval(text)) + except (ValueError, SyntaxError): + return value + return value + + +def sheet_draft_from_sheet(sheet: Any) -> SheetDraft: + """A SheetDraft from a sheet read from Egeria (pyegeria.view.dashboards.EgeriaDashboardSheet).""" + notes = list(getattr(sheet, "breaks", None) or []) + placements = [] + for i, p in enumerate(sheet.placements, start=1): + kind = getattr(p, "kind", "report") + if getattr(p, "note", None): + notes.append(f"placement {i} ({p.ref}): {p.note}") + if kind == "text" or p.content is not None: + placements.append(SheetPlacement(text=p.content or "", name=p.ref, span=p.span, emphasis=p.emphasis, + perspectives=list(p.perspectives or []), detail_spec=p.detail_spec)) + else: + if kind == "missing": + notes.append(f"placement {i} ({p.ref}): its Report link is missing in Egeria; kept by name") + placements.append(SheetPlacement(report=p.ref, span=p.span, emphasis=p.emphasis, + perspectives=list(p.perspectives or []), detail_spec=p.detail_spec)) + return SheetDraft(name=sheet.name, heading=sheet.heading, description=sheet.description or "", + family=sheet.family, placements=placements, guid=getattr(sheet, "guid", None), notes=notes) + + +async def async_load_held(kind: str, label: str, client: Any) -> Union[SpecDraft, ReportDraft, SheetDraft]: + """The held artifact `label` as a draft, read from Egeria as the caller, with its Egeria version. + - spec: the ReportSpec element `ReportSpec::