From 431499933bca498ccdb8b6c6d071306d5efe68c3 Mon Sep 17 00:00:00 2001 From: Peter Tomko Date: Sat, 21 Nov 2026 14:12:07 +0100 Subject: [PATCH] feat(gooddata-sdk): AI Knowledge folders as a declarative YAML layout catalog_ai_knowledge stores the organization's AI Knowledge as files: one directory per folder, nested like the folders, each with folder.yaml (name, description, purpose, extraction: model, summary and section instructions, fields, labels; access: all, inherit or workspaces) and documents.yaml (file name, title, and an optional local path or https source). Content never goes into the YAML. load_and_put_declarative_ai_knowledge makes AI Knowledge match the layout: folders are matched by their path of names and created or updated, a missing document is uploaded in parts (any size) or imported by the server, a document in another folder is moved, and delete_missing removes folders that are no longer listed. A subfolder inherits extraction and access unless it sets its own. GoodDataApiClient.request sends authenticated requests to endpoints the generated client does not cover. jira: trivial risk: low Co-Authored-By: Claude Opus 5.5 --- .../gooddata-sdk/src/gooddata_sdk/__init__.py | 28 ++ .../catalog/ai_knowledge/__init__.py | 1 + .../catalog/ai_knowledge/declarative_model.py | 349 ++++++++++++++++++ .../catalog/ai_knowledge/service.py | 348 +++++++++++++++++ .../gooddata-sdk/src/gooddata_sdk/client.py | 41 ++ packages/gooddata-sdk/src/gooddata_sdk/sdk.py | 6 + .../unit_tests/test_ai_knowledge_layout.py | 281 ++++++++++++++ 7 files changed, 1054 insertions(+) create mode 100644 packages/gooddata-sdk/src/gooddata_sdk/catalog/ai_knowledge/__init__.py create mode 100644 packages/gooddata-sdk/src/gooddata_sdk/catalog/ai_knowledge/declarative_model.py create mode 100644 packages/gooddata-sdk/src/gooddata_sdk/catalog/ai_knowledge/service.py create mode 100644 packages/gooddata-sdk/tests/catalog/unit_tests/test_ai_knowledge_layout.py diff --git a/packages/gooddata-sdk/src/gooddata_sdk/__init__.py b/packages/gooddata-sdk/src/gooddata_sdk/__init__.py index fd9545ea1..146dd0803 100644 --- a/packages/gooddata-sdk/src/gooddata_sdk/__init__.py +++ b/packages/gooddata-sdk/src/gooddata_sdk/__init__.py @@ -14,6 +14,18 @@ if TYPE_CHECKING: # Re-stated eagerly so that type checkers and IDEs resolve the lazy re-exports below. + from gooddata_sdk.catalog.ai_knowledge.declarative_model import ( + CatalogDeclarativeAIKnowledge, + CatalogDeclarativeKnowledgeDocument, + CatalogDeclarativeKnowledgeExtraction, + CatalogDeclarativeKnowledgeField, + CatalogDeclarativeKnowledgeFolder, + ) + from gooddata_sdk.catalog.ai_knowledge.service import ( + AIKnowledgeApiError, + CatalogAIKnowledgeService, + CatalogAIKnowledgeSyncResult, + ) from gooddata_sdk.catalog.ai_lake.service import ( CatalogAILakeOperation, CatalogAILakeOperationError, @@ -385,6 +397,7 @@ # declarative model and generated API model) even when the caller wants a single class, so they # are resolved on first access instead (PEP 562). Do not edit by hand -- see the script. _LAZY_IMPORTS: dict[str, str] = { + "AIKnowledgeApiError": "gooddata_sdk.catalog.ai_knowledge.service", "AbsoluteDateFilter": "gooddata_sdk.compute.model.filter", "AllMetricValueFilter": "gooddata_sdk.compute.model.filter", "AllTimeDateFilter": "gooddata_sdk.compute.model.filter", @@ -395,6 +408,8 @@ "BareExecutionResponse": "gooddata_sdk.compute.model.execution", "BasicCredentials": "gooddata_sdk.catalog.entity", "BoundedFilter": "gooddata_sdk.compute.model.filter", + "CatalogAIKnowledgeService": "gooddata_sdk.catalog.ai_knowledge.service", + "CatalogAIKnowledgeSyncResult": "gooddata_sdk.catalog.ai_knowledge.service", "CatalogAILakeOperation": "gooddata_sdk.catalog.ai_lake.service", "CatalogAILakeOperationError": "gooddata_sdk.catalog.ai_lake.service", "CatalogAILakeService": "gooddata_sdk.catalog.ai_lake.service", @@ -435,6 +450,7 @@ "CatalogDataSourceVertica": "gooddata_sdk.catalog.data_source.entity_model.data_source", "CatalogDataset": "gooddata_sdk.catalog.workspace.entity_model.content_objects.dataset", "CatalogDatasetWorkspaceDataFilterIdentifier": "gooddata_sdk.catalog.identifier", + "CatalogDeclarativeAIKnowledge": "gooddata_sdk.catalog.ai_knowledge.declarative_model", "CatalogDeclarativeAggregatedFact": "gooddata_sdk.catalog.workspace.declarative_model.workspace.logical_model.dataset.dataset", "CatalogDeclarativeAnalyticalDashboardIdentifier": "gooddata_sdk.catalog.identifier", "CatalogDeclarativeAnalytics": "gooddata_sdk.catalog.workspace.declarative_model.workspace.analytics_model.analytics_model", @@ -455,6 +471,10 @@ "CatalogDeclarativeExportTemplate": "gooddata_sdk.catalog.organization.layout.export_template", "CatalogDeclarativeFact": "gooddata_sdk.catalog.workspace.declarative_model.workspace.logical_model.dataset.dataset", "CatalogDeclarativeFilterView": "gooddata_sdk.catalog.workspace.declarative_model.workspace.workspace", + "CatalogDeclarativeKnowledgeDocument": "gooddata_sdk.catalog.ai_knowledge.declarative_model", + "CatalogDeclarativeKnowledgeExtraction": "gooddata_sdk.catalog.ai_knowledge.declarative_model", + "CatalogDeclarativeKnowledgeField": "gooddata_sdk.catalog.ai_knowledge.declarative_model", + "CatalogDeclarativeKnowledgeFolder": "gooddata_sdk.catalog.ai_knowledge.declarative_model", "CatalogDeclarativeLabel": "gooddata_sdk.catalog.workspace.declarative_model.workspace.logical_model.dataset.dataset", "CatalogDeclarativeLdm": "gooddata_sdk.catalog.workspace.declarative_model.workspace.logical_model.ldm", "CatalogDeclarativeMemoryItem": "gooddata_sdk.catalog.workspace.declarative_model.workspace.analytics_model.analytics_model", @@ -659,6 +679,7 @@ __all__ = [ "__version__", + "AIKnowledgeApiError", "AbsoluteDateFilter", "AllMetricValueFilter", "AllTimeDateFilter", @@ -669,6 +690,8 @@ "BareExecutionResponse", "BasicCredentials", "BoundedFilter", + "CatalogAIKnowledgeService", + "CatalogAIKnowledgeSyncResult", "CatalogAILakeOperation", "CatalogAILakeOperationError", "CatalogAILakeService", @@ -709,6 +732,7 @@ "CatalogDataSourceVertica", "CatalogDataset", "CatalogDatasetWorkspaceDataFilterIdentifier", + "CatalogDeclarativeAIKnowledge", "CatalogDeclarativeAggregatedFact", "CatalogDeclarativeAnalyticalDashboardIdentifier", "CatalogDeclarativeAnalytics", @@ -729,6 +753,10 @@ "CatalogDeclarativeExportTemplate", "CatalogDeclarativeFact", "CatalogDeclarativeFilterView", + "CatalogDeclarativeKnowledgeDocument", + "CatalogDeclarativeKnowledgeExtraction", + "CatalogDeclarativeKnowledgeField", + "CatalogDeclarativeKnowledgeFolder", "CatalogDeclarativeLabel", "CatalogDeclarativeLdm", "CatalogDeclarativeMemoryItem", diff --git a/packages/gooddata-sdk/src/gooddata_sdk/catalog/ai_knowledge/__init__.py b/packages/gooddata-sdk/src/gooddata_sdk/catalog/ai_knowledge/__init__.py new file mode 100644 index 000000000..efe7c60c8 --- /dev/null +++ b/packages/gooddata-sdk/src/gooddata_sdk/catalog/ai_knowledge/__init__.py @@ -0,0 +1 @@ +# (C) 2026 GoodData Corporation diff --git a/packages/gooddata-sdk/src/gooddata_sdk/catalog/ai_knowledge/declarative_model.py b/packages/gooddata-sdk/src/gooddata_sdk/catalog/ai_knowledge/declarative_model.py new file mode 100644 index 000000000..0216027b3 --- /dev/null +++ b/packages/gooddata-sdk/src/gooddata_sdk/catalog/ai_knowledge/declarative_model.py @@ -0,0 +1,349 @@ +# (C) 2026 GoodData Corporation +"""AI Knowledge as files: one directory per folder, nested like the folders. + +Layout under `gooddata_layouts//ai_knowledge/`:: + + documents.yaml # documents not in any folder + contracts/ + folder.yaml # name, description, purpose, extraction, access + documents.yaml # the folder's documents + files/pricing.pdf # optional: a document's file, referenced by `path` + customers/ + folder.yaml # a subfolder: inherits extraction and access unless it sets its own + documents.yaml + +A directory is a folder when it holds `folder.yaml`; other directories (such as `files/`) are ignored. The folder's +name is the `name` in `folder.yaml`; the directory name is only a file-system safe form of it. + +`folder.yaml`:: + + name: Contracts + description: Signed customer contracts + purpose: documents # documents | business_context | investigations + extraction: # or `inherit` in a subfolder + model: gpt-4.1-mini # optional: the LLM to use; the organization default otherwise + summary_instruction: Summarise the commercial terms in three sentences. + instruction: Focus on obligations of the customer. + labels: [contract, legal] + fields: + - name: Renewal date + instruction: When does the contract renew? ISO date. + access: all # all | inherit | {workspaces: [sales, finance]} + +`documents.yaml` lists documents, never their content:: + + - file: pricing-policy.pdf # the document's name in AI Knowledge + title: Pricing policy # optional + path: files/pricing-policy.pdf # optional: upload this file, relative to the folder's directory + source: https://example.com/p.pdf # optional: let the server import the file from this address + +A document with neither `path` nor `source` is only described: it must already be in AI Knowledge. +""" + +from __future__ import annotations + +import re +from pathlib import Path +from typing import Any, Literal + +from attrs import define, field + +from gooddata_sdk.utils import read_layout_from_file, write_layout_to_file + +AI_KNOWLEDGE_FOLDER = "ai_knowledge" +FOLDER_FILE = "folder.yaml" +DOCUMENTS_FILE = "documents.yaml" + +INHERIT = "inherit" +ALL_WORKSPACES = "all" + +FolderPurpose = Literal["documents", "business_context", "investigations"] + + +@define(kw_only=True) +class CatalogDeclarativeKnowledgeField: + """A section the LLM writes for every document of a folder.""" + + name: str + instruction: str = "" + + +@define(kw_only=True) +class CatalogDeclarativeKnowledgeExtraction: + """What the LLM writes about each document of a folder, and with which model.""" + + model: str | None = None + summary_instruction: str = "" + instruction: str = "" + labels: list[str] = field(factory=list) + fields: list[CatalogDeclarativeKnowledgeField] = field(factory=list) + + def to_yaml(self) -> dict[str, Any]: + data: dict[str, Any] = {} + if self.model: + data["model"] = self.model + if self.summary_instruction: + data["summary_instruction"] = self.summary_instruction + if self.instruction: + data["instruction"] = self.instruction + if self.labels: + data["labels"] = list(self.labels) + if self.fields: + data["fields"] = [{"name": f.name, "instruction": f.instruction} for f in self.fields] + return data + + @classmethod + def from_yaml(cls, data: dict[str, Any] | None) -> CatalogDeclarativeKnowledgeExtraction: + data = data or {} + return cls( + model=data.get("model") or None, + summary_instruction=str(data.get("summary_instruction") or ""), + instruction=str(data.get("instruction") or ""), + labels=[str(label) for label in data.get("labels") or []], + fields=[ + CatalogDeclarativeKnowledgeField(name=str(f["name"]), instruction=str(f.get("instruction") or "")) + for f in data.get("fields") or [] + ], + ) + + def to_api(self) -> dict[str, Any]: + return { + "model": self.model, + "summaryInstruction": self.summary_instruction, + "instruction": self.instruction, + "labels": list(self.labels), + "fields": [{"name": f.name, "instruction": f.instruction} for f in self.fields], + } + + @classmethod + def from_api(cls, data: dict[str, Any] | None) -> CatalogDeclarativeKnowledgeExtraction: + data = data or {} + return cls( + model=data.get("model") or None, + summary_instruction=str(data.get("summaryInstruction") or ""), + instruction=str(data.get("instruction") or ""), + labels=[str(label) for label in data.get("labels") or []], + fields=[ + CatalogDeclarativeKnowledgeField(name=str(f["name"]), instruction=str(f.get("instruction") or "")) + for f in data.get("fields") or [] + ], + ) + + +@define(kw_only=True) +class CatalogDeclarativeKnowledgeDocument: + """A document of AI Knowledge, described by its name; its file is optional.""" + + file: str + title: str | None = None + #: A local file to upload, relative to the directory of the folder's `documents.yaml`. + path: str | None = None + #: An https address the server imports the file from. + source: str | None = None + + def to_yaml(self) -> dict[str, Any]: + data: dict[str, Any] = {"file": self.file} + for key in ("title", "path", "source"): + if getattr(self, key): + data[key] = getattr(self, key) + return data + + @classmethod + def from_yaml(cls, data: dict[str, Any] | str) -> CatalogDeclarativeKnowledgeDocument: + if isinstance(data, str): + return cls(file=data) + return cls( + file=str(data["file"]), + title=data.get("title") or None, + path=data.get("path") or None, + source=data.get("source") or None, + ) + + +@define(kw_only=True) +class CatalogDeclarativeKnowledgeFolder: + """A folder of AI Knowledge with its documents and subfolders. + + `extraction` None and `access` "inherit" take the parent's; a top-level folder cannot inherit. + """ + + name: str + description: str | None = None + purpose: FolderPurpose = "documents" + extraction: CatalogDeclarativeKnowledgeExtraction | None = None + #: "all", "inherit", or the ids of the workspaces that see the folder (with their child workspaces). + access: str | list[str] = ALL_WORKSPACES + documents: list[CatalogDeclarativeKnowledgeDocument] = field(factory=list) + folders: list[CatalogDeclarativeKnowledgeFolder] = field(factory=list) + #: The directory the folder was loaded from; local document paths are relative to it. + directory: Path | None = field(default=None, eq=False) + + @property + def inherits_extraction(self) -> bool: + return self.extraction is None + + @property + def inherits_access(self) -> bool: + return self.access == INHERIT + + def walk( + self, parent_path: tuple[str, ...] = () + ) -> list[tuple[tuple[str, ...], CatalogDeclarativeKnowledgeFolder]]: + """This folder and every subfolder with its path of names, parents first.""" + path = (*parent_path, self.name) + result = [(path, self)] + for child in self.folders: + result += child.walk(path) + return result + + def folder_yaml(self) -> dict[str, Any]: + data: dict[str, Any] = {"name": self.name} + if self.description: + data["description"] = self.description + if self.purpose != "documents": + data["purpose"] = self.purpose + data["extraction"] = INHERIT if self.extraction is None else self.extraction.to_yaml() + data["access"] = self.access if isinstance(self.access, str) else {"workspaces": list(self.access)} + return data + + @classmethod + def from_folder_yaml( + cls, data: dict[str, Any], *, top_level: bool, directory: Path | None = None + ) -> CatalogDeclarativeKnowledgeFolder: + if not data or not data.get("name"): + raise ValueError(f"{directory or 'folder'}/{FOLDER_FILE} needs a name") + extraction_data = data.get("extraction", {} if top_level else INHERIT) + extraction = ( + None + if extraction_data == INHERIT + else CatalogDeclarativeKnowledgeExtraction.from_yaml(extraction_data or {}) + ) + access_data = data.get("access", ALL_WORKSPACES if top_level else INHERIT) + access: str | list[str] + if isinstance(access_data, dict): + access = [str(w) for w in access_data.get("workspaces") or []] + elif isinstance(access_data, list): + access = [str(w) for w in access_data] + elif access_data in (ALL_WORKSPACES, INHERIT): + access = str(access_data) + else: + raise ValueError(f"{directory}/{FOLDER_FILE}: access is all, inherit or {{workspaces: [...]}}") + if top_level and (extraction is None or access == INHERIT): + raise ValueError(f"{directory}/{FOLDER_FILE}: a top-level folder has no parent to inherit from") + return cls( + name=str(data["name"]), + description=data.get("description") or None, + purpose=data.get("purpose") or "documents", + extraction=extraction, + access=access, + directory=directory, + ) + + def store_to_disk(self, directory: Path) -> None: + directory.mkdir(parents=True, exist_ok=True) + write_layout_to_file(directory / FOLDER_FILE, self.folder_yaml()) + _write_documents(directory, self.documents) + _store_folders(directory, self.folders) + + @classmethod + def load_from_disk(cls, directory: Path, *, top_level: bool) -> CatalogDeclarativeKnowledgeFolder: + folder = cls.from_folder_yaml( + read_layout_from_file(directory / FOLDER_FILE), top_level=top_level, directory=directory + ) + folder.documents = _read_documents(directory) + folder.folders = _load_folders(directory) + return folder + + +@define(kw_only=True) +class CatalogDeclarativeAIKnowledge: + """The AI Knowledge of an organization: its folder tree and the documents in no folder.""" + + folders: list[CatalogDeclarativeKnowledgeFolder] = field(factory=list) + documents: list[CatalogDeclarativeKnowledgeDocument] = field(factory=list) + directory: Path | None = field(default=None, eq=False) + + def walk(self) -> list[tuple[tuple[str, ...], CatalogDeclarativeKnowledgeFolder]]: + result: list[tuple[tuple[str, ...], CatalogDeclarativeKnowledgeFolder]] = [] + for folder in self.folders: + result += folder.walk() + return result + + def store_to_disk(self, organization_folder: Path) -> None: + """Write the layout. Files of documents (`path`) and other files in the directories are kept.""" + directory = organization_folder / AI_KNOWLEDGE_FOLDER + directory.mkdir(parents=True, exist_ok=True) + _write_documents(directory, self.documents) + _store_folders(directory, self.folders) + + @classmethod + def load_from_disk(cls, organization_folder: Path) -> CatalogDeclarativeAIKnowledge: + directory = organization_folder / AI_KNOWLEDGE_FOLDER + if not directory.is_dir(): + raise ValueError(f"No AI Knowledge layout in {directory}") + return cls( + folders=_load_folders(directory, top_level=True), documents=_read_documents(directory), directory=directory + ) + + +def directory_name(name: str) -> str: + """A file-system safe directory name for a folder name.""" + safe = re.sub(r"[^\w.-]+", "_", name.strip(), flags=re.UNICODE).strip("._") or "folder" + return safe[:100] + + +def _store_folders(directory: Path, folders: list[CatalogDeclarativeKnowledgeFolder]) -> None: + used: set[str] = set() + keep: set[str] = set() + for folder in folders: + name = directory_name(folder.name) + candidate, n = name, 2 + while candidate.lower() in used: + candidate, n = f"{name}_{n}", n + 1 + used.add(candidate.lower()) + keep.add(candidate) + folder.store_to_disk(directory / candidate) + # A folder deleted since the last store: drop its layout files, keep any other file a user put there. + for child in directory.iterdir(): + if child.is_dir() and child.name not in keep and (child / FOLDER_FILE).is_file(): + _remove_layout(child) + + +def _remove_layout(directory: Path) -> None: + for child in directory.iterdir(): + if child.is_dir() and (child / FOLDER_FILE).is_file(): + _remove_layout(child) + for name in (FOLDER_FILE, DOCUMENTS_FILE): + (directory / name).unlink(missing_ok=True) + if not any(directory.iterdir()): + directory.rmdir() + + +def _load_folders(directory: Path, *, top_level: bool = False) -> list[CatalogDeclarativeKnowledgeFolder]: + return [ + CatalogDeclarativeKnowledgeFolder.load_from_disk(child, top_level=top_level) + for child in sorted(directory.iterdir()) + if child.is_dir() and (child / FOLDER_FILE).is_file() + ] + + +def _write_documents(directory: Path, documents: list[CatalogDeclarativeKnowledgeDocument]) -> None: + path = directory / DOCUMENTS_FILE + # The server knows no local file or import address: keep the ones already written for the same document. + known = {d.file.lower(): d for d in _read_documents(directory)} + for document in documents: + previous = known.get(document.file.lower()) + if previous is not None: + document.path = document.path or previous.path + document.source = document.source or previous.source + if documents: + write_layout_to_file(path, [d.to_yaml() for d in sorted(documents, key=lambda d: d.file.lower())]) + else: + path.unlink(missing_ok=True) + + +def _read_documents(directory: Path) -> list[CatalogDeclarativeKnowledgeDocument]: + path = directory / DOCUMENTS_FILE + if not path.is_file(): + return [] + return [CatalogDeclarativeKnowledgeDocument.from_yaml(item) for item in read_layout_from_file(path) or []] diff --git a/packages/gooddata-sdk/src/gooddata_sdk/catalog/ai_knowledge/service.py b/packages/gooddata-sdk/src/gooddata_sdk/catalog/ai_knowledge/service.py new file mode 100644 index 000000000..fd81d3c5f --- /dev/null +++ b/packages/gooddata-sdk/src/gooddata_sdk/catalog/ai_knowledge/service.py @@ -0,0 +1,348 @@ +# (C) 2026 GoodData Corporation +"""AI Knowledge of the organization as a declarative layout: folders, their settings, and their documents. + +`store_declarative_ai_knowledge` writes the folder tree to YAML (see `declarative_model`); edit it, add documents +with a local `path` or a `source` address, and `load_and_put_declarative_ai_knowledge` makes AI Knowledge match. +Document content never goes into the YAML: a file is uploaded in parts (any size) or imported by the server. +""" + +from __future__ import annotations + +import mimetypes +import time +from collections.abc import Callable +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from typing import Any + +import requests +from attrs import define, field + +from gooddata_sdk.catalog.ai_knowledge.declarative_model import ( + ALL_WORKSPACES, + CatalogDeclarativeAIKnowledge, + CatalogDeclarativeKnowledgeDocument, + CatalogDeclarativeKnowledgeExtraction, + CatalogDeclarativeKnowledgeFolder, +) +from gooddata_sdk.catalog.catalog_service_base import CatalogServiceBase +from gooddata_sdk.client import GoodDataApiClient + +_BASE = "/api/v1/ai/organization/knowledge" +_PAGE_SIZE = 500 +_PART_ATTEMPTS = 3 + + +class AIKnowledgeApiError(RuntimeError): + """AI Knowledge refused a request.""" + + def __init__(self, response: requests.Response) -> None: + try: + detail = response.json().get("detail", response.text) + except ValueError: + detail = response.text + super().__init__(f"{response.request.method} {response.url} -> HTTP {response.status_code}: {detail}") + self.status_code = response.status_code + + +@define(kw_only=True) +class CatalogAIKnowledgeSyncResult: + """What `put_declarative_ai_knowledge` changed.""" + + folders_created: list[str] = field(factory=list) + folders_updated: list[str] = field(factory=list) + folders_deleted: list[str] = field(factory=list) + documents_uploaded: list[str] = field(factory=list) + documents_imported: list[str] = field(factory=list) + documents_moved: list[str] = field(factory=list) + documents_missing: list[str] = field(factory=list) + """Listed in the layout without a `path` or `source`, and not in AI Knowledge.""" + + +class CatalogAIKnowledgeService(CatalogServiceBase): + """Folders and documents of the organization's AI Knowledge. Every call needs organization admin rights.""" + + def __init__(self, api_client: GoodDataApiClient) -> None: + super().__init__(api_client) + + # Declarative layout + + def get_declarative_ai_knowledge(self) -> CatalogDeclarativeAIKnowledge: + """The folder tree with settings and the list of documents. Files are not downloaded.""" + folders = self.list_folders() + by_id = {f["id"]: f for f in folders} + nodes = {f["id"]: _folder_from_api(f) for f in folders} + layout = CatalogDeclarativeAIKnowledge() + for folder in sorted(folders, key=lambda f: str(f["name"]).lower()): + node = nodes[folder["id"]] + parent_id = folder.get("parentId") + if parent_id and parent_id in nodes: + nodes[parent_id].folders.append(node) + else: + layout.folders.append(node) + for document in self.list_documents(): + item = CatalogDeclarativeKnowledgeDocument(file=document["filename"], title=document.get("title") or None) + folder_id = document.get("folderId") + if folder_id and folder_id in by_id: + nodes[folder_id].documents.append(item) + else: + layout.documents.append(item) + return layout + + def store_declarative_ai_knowledge(self, layout_root_path: Path = Path.cwd()) -> None: + """Write the layout to `/gooddata_layouts//ai_knowledge/`.""" + self.get_declarative_ai_knowledge().store_to_disk(self.layout_organization_folder(layout_root_path)) + + def load_declarative_ai_knowledge(self, layout_root_path: Path = Path.cwd()) -> CatalogDeclarativeAIKnowledge: + return CatalogDeclarativeAIKnowledge.load_from_disk(self.layout_organization_folder(layout_root_path)) + + def load_and_put_declarative_ai_knowledge( + self, + layout_root_path: Path = Path.cwd(), + *, + replace_documents: bool = False, + delete_missing: bool = False, + ) -> CatalogAIKnowledgeSyncResult: + """Make AI Knowledge match the layout stored under `layout_root_path`. See `put_declarative_ai_knowledge`.""" + return self.put_declarative_ai_knowledge( + self.load_declarative_ai_knowledge(layout_root_path), + replace_documents=replace_documents, + delete_missing=delete_missing, + ) + + def put_declarative_ai_knowledge( + self, + layout: CatalogDeclarativeAIKnowledge, + *, + replace_documents: bool = False, + delete_missing: bool = False, + ) -> CatalogAIKnowledgeSyncResult: + """Create and update folders as in the layout, then upload, import or move the documents it lists. + + Folders are matched by their path of names. A document is matched by its file name: a missing one is + uploaded from its `path` or imported from its `source`; one in another folder is moved. With + `replace_documents`, documents that have a `path` or `source` are sent again even when they exist. + + With `delete_missing`, folders not in the layout are deleted with their documents and subfolders. Documents + are never deleted otherwise. + """ + result = CatalogAIKnowledgeSyncResult() + existing = {tuple(f.get("path") or [f["name"]]): f for f in self.list_folders()} + ids: dict[tuple[str, ...], str] = {} + + for path, folder in layout.walk(): + parent_id = ids.get(path[:-1]) if len(path) > 1 else None + body = _folder_to_api(folder, parent_id) + current = existing.get(path) + if current is None: + created = self._json("POST", "/folders", json=body) + ids[path] = created["id"] + result.folders_created.append("/".join(path)) + else: + ids[path] = current["id"] + if _folder_differs(current, body): + self._json("PATCH", f"/folders/{current['id']}", json=body) + result.folders_updated.append("/".join(path)) + + documents = {d["filename"].lower(): d for d in self.list_documents()} + work: list[tuple[CatalogDeclarativeKnowledgeDocument, str | None, Path | None]] = [ + (d, None, layout.directory) for d in layout.documents + ] + for path, folder in layout.walk(): + work += [(d, ids[path], folder.directory) for d in folder.documents] + for document, folder_id, directory in work: + current = documents.get(document.file.lower()) + has_content = bool(document.path or document.source) + if current is not None and not (replace_documents and has_content): + if (current.get("folderId") or None) != folder_id: + self._json("PATCH", f"/documents/{current['id']}", json={"folderId": folder_id}) + result.documents_moved.append(document.file) + if document.title and document.title != current.get("title"): + self._json("PATCH", f"/documents/{current['id']}", json={"title": document.title}) + continue + if document.path: + local = Path(document.path) + if not local.is_absolute() and directory is not None: + local = directory / local + self.upload_document( + local, + filename=document.file, + folder_id=folder_id, + title=document.title, + replace=current is not None, + ) + result.documents_uploaded.append(document.file) + elif document.source: + self.import_document( + document.source, + filename=document.file, + folder_id=folder_id, + title=document.title, + replace=current is not None, + ) + result.documents_imported.append(document.file) + else: + result.documents_missing.append(document.file) + + if delete_missing: + wanted = set(ids) + stale = [p for p in existing if p not in wanted and p[:-1] in wanted | {()}] + for path in sorted(stale, key=len, reverse=True): + self._request("DELETE", f"/folders/{existing[path]['id']}", params={"documents": "delete"}) + result.folders_deleted.append("/".join(path)) + return result + + # Folders and documents + + def list_folders(self) -> list[dict[str, Any]]: + return list(self._json("GET", "/folders").get("folders", [])) + + def list_documents(self) -> list[dict[str, Any]]: + """Organization-level documents, every page.""" + documents: list[dict[str, Any]] = [] + token: str | None = None + while True: + params: dict[str, Any] = {"size": _PAGE_SIZE} + if token: + params["pageToken"] = token + page = self._json("GET", "/documents", params=params) + documents += page.get("documents", []) + token = page.get("nextPageToken") + if not token: + return documents + + def upload_document( + self, + path: Path, + *, + filename: str | None = None, + folder_id: str | None = None, + title: str | None = None, + content_type: str | None = None, + replace: bool = False, + parallel_parts: int = 3, + on_progress: Callable[[int, int], None] | None = None, + ) -> dict[str, Any]: + """Upload a local file of any size in parts. Returns the queued document (id, filename, processingStatus).""" + name = filename or path.name + size = path.stat().st_size + upload = self._json( + "POST", + "/uploads", + json={ + "filename": name, + "contentType": content_type or mimetypes.guess_type(name)[0] or "application/octet-stream", + "size": size, + "title": title, + "folderId": folder_id, + "replace": replace, + }, + ) + upload_id, part_size, part_count = upload["uploadId"], upload["partSize"], upload["partCount"] + sent = [0] + + def send(number: int) -> None: + with path.open("rb") as fh: + fh.seek((number - 1) * part_size) + data = fh.read(part_size) + for attempt in range(_PART_ATTEMPTS): + response = self._request("PUT", f"/uploads/{upload_id}/parts/{number}", data=data, check=False) + if response.status_code < 400: + break + if response.status_code < 500 or attempt == _PART_ATTEMPTS - 1: + raise AIKnowledgeApiError(response) + time.sleep(2**attempt) + sent[0] += len(data) + if on_progress is not None: + on_progress(sent[0], size) + + try: + with ThreadPoolExecutor(max_workers=max(1, parallel_parts)) as pool: + list(pool.map(send, range(1, part_count + 1))) + return self._json("POST", f"/uploads/{upload_id}/complete") + except Exception: + self._request("DELETE", f"/uploads/{upload_id}", check=False) + raise + + def import_document( + self, + url: str, + *, + filename: str | None = None, + folder_id: str | None = None, + title: str | None = None, + replace: bool = False, + ) -> dict[str, Any]: + """Let the server copy a file from an https address on a public host. Returns the queued document.""" + return self._json( + "POST", + "/imports", + json={"url": url, "filename": filename, "folderId": folder_id, "title": title, "replace": replace}, + ) + + def put_transcript(self, document_id: str, path: Path) -> None: + """Add a WebVTT or SubRip transcript to an audio or video document, which makes it searchable.""" + with path.open("rb") as fh: + self._request("PUT", f"/documents/{document_id}/transcript", files={"file": (path.name, fh, "text/vtt")}) + + # HTTP + + def _request( + self, + method: str, + endpoint: str, + *, + check: bool = True, + **kwargs: Any, + ) -> requests.Response: + response = self._client.request(method, f"{_BASE}{endpoint}", **kwargs) + if check and response.status_code >= 400: + raise AIKnowledgeApiError(response) + return response + + def _json(self, method: str, endpoint: str, **kwargs: Any) -> dict[str, Any]: + response = self._request(method, endpoint, **kwargs) + return response.json() if response.content else {} + + +def _folder_from_api(data: dict[str, Any]) -> CatalogDeclarativeKnowledgeFolder: + access = data.get("access") + top_level = not data.get("parentId") + return CatalogDeclarativeKnowledgeFolder( + name=data["name"], + description=data.get("description") or None, + purpose=data.get("purpose") or "documents", + extraction=None + if data.get("inheritExtraction") and not top_level + else CatalogDeclarativeKnowledgeExtraction.from_api(data.get("extraction")), + access="inherit" + if data.get("inheritAccess") and not top_level + else (ALL_WORKSPACES if access is None else list(access.get("workspaceIds") or [])), + ) + + +def _folder_to_api(folder: CatalogDeclarativeKnowledgeFolder, parent_id: str | None) -> dict[str, Any]: + body: dict[str, Any] = { + "name": folder.name, + "description": folder.description, + "purpose": folder.purpose, + "parentId": parent_id, + "inheritExtraction": folder.inherits_extraction and parent_id is not None, + "inheritAccess": folder.inherits_access and parent_id is not None, + } + if folder.extraction is not None: + body["extraction"] = folder.extraction.to_api() + if not folder.inherits_access: + body["access"] = None if folder.access == ALL_WORKSPACES else {"workspaceIds": list(folder.access)} + return body + + +def _folder_differs(current: dict[str, Any], body: dict[str, Any]) -> bool: + for key in ("description", "purpose", "parentId", "inheritExtraction", "inheritAccess"): + if (current.get(key) or None) != (body.get(key) or None): + return True + if "extraction" in body and CatalogDeclarativeKnowledgeExtraction.from_api( + current.get("extraction") + ) != CatalogDeclarativeKnowledgeExtraction.from_api(body["extraction"]): + return True + return "access" in body and (current.get("access") or None) != (body["access"] or None) diff --git a/packages/gooddata-sdk/src/gooddata_sdk/client.py b/packages/gooddata-sdk/src/gooddata_sdk/client.py index 8a17cbfe6..d9199d87c 100644 --- a/packages/gooddata-sdk/src/gooddata_sdk/client.py +++ b/packages/gooddata-sdk/src/gooddata_sdk/client.py @@ -7,6 +7,7 @@ import os from dataclasses import dataclass, field from pathlib import Path +from typing import Any import gooddata_api_client as api_client import requests @@ -248,6 +249,46 @@ def do_request( else: raise NotImplementedError("Currently only supports the POST method.") + def request( + self, + method: str, + endpoint: str, + *, + json: Any = None, + data: bytes | None = None, + params: dict[str, Any] | None = None, + files: dict[str, Any] | None = None, + headers: dict[str, str] | None = None, + timeout: float = 300.0, + ) -> requests.Response: + """Send an authenticated request to an endpoint the generated client does not cover, e.g. AI Knowledge. + + Args: + method (str): HTTP method, e.g. "GET" or "PUT". + endpoint (str): Path below the host, e.g. "/api/v1/ai/organization/knowledge/folders". + json (Any): JSON body. + data (bytes | None): Raw body, sent as application/octet-stream unless headers say otherwise. + params (dict[str, Any] | None): Query parameters. + files (dict[str, Any] | None): Multipart files, as for `requests`. + headers (dict[str, str] | None): Extra headers. + timeout (float): Seconds to wait for the server. + + Returns: + requests.Response: The response; the caller checks its status. + """ + url = f"{self._hostname.rstrip('/')}/{endpoint.lstrip('/')}" + all_headers = { + "Authorization": f"Bearer {self._token}", + "X-Requested-With": "XMLHttpRequest", + **self._custom_headers, + **(headers or {}), + } + if data is not None and "Content-Type" not in all_headers: + all_headers["Content-Type"] = "application/octet-stream" + return self._session.request( + method, url, json=json, data=data, params=params, files=files, headers=all_headers, timeout=timeout + ) + @staticmethod def _set_default_headers(headers: dict) -> None: headers["X-Requested-With"] = "XMLHttpRequest" diff --git a/packages/gooddata-sdk/src/gooddata_sdk/sdk.py b/packages/gooddata-sdk/src/gooddata_sdk/sdk.py index 190613ffa..ac51cf898 100644 --- a/packages/gooddata-sdk/src/gooddata_sdk/sdk.py +++ b/packages/gooddata-sdk/src/gooddata_sdk/sdk.py @@ -3,6 +3,7 @@ from pathlib import Path +from gooddata_sdk.catalog.ai_knowledge.service import CatalogAIKnowledgeService from gooddata_sdk.catalog.ai_lake.service import CatalogAILakeService from gooddata_sdk.catalog.appearance.service import CatalogAppearanceService from gooddata_sdk.catalog.data_source.service import CatalogDataSourceService @@ -96,6 +97,7 @@ def __init__(self, client: GoodDataApiClient) -> None: self._support = SupportService(self._client) self._catalog_permission = CatalogPermissionService(self._client) self._catalog_ai_lake = CatalogAILakeService(self._client) + self._catalog_ai_knowledge = CatalogAIKnowledgeService(self._client) self._export = ExportService(self._client) @property @@ -150,6 +152,10 @@ def export(self) -> ExportService: def catalog_ai_lake(self) -> CatalogAILakeService: return self._catalog_ai_lake + @property + def catalog_ai_knowledge(self) -> CatalogAIKnowledgeService: + return self._catalog_ai_knowledge + @property def client(self) -> GoodDataApiClient: return self._client diff --git a/packages/gooddata-sdk/tests/catalog/unit_tests/test_ai_knowledge_layout.py b/packages/gooddata-sdk/tests/catalog/unit_tests/test_ai_knowledge_layout.py new file mode 100644 index 000000000..4574c784b --- /dev/null +++ b/packages/gooddata-sdk/tests/catalog/unit_tests/test_ai_knowledge_layout.py @@ -0,0 +1,281 @@ +# (C) 2026 GoodData Corporation +"""AI Knowledge as YAML: the folder tree on disk, and syncing it with the AI Knowledge API.""" + +from __future__ import annotations + +import json +import re +import uuid +from pathlib import Path +from typing import Any +from unittest.mock import MagicMock + +import pytest +import requests +import yaml +from gooddata_sdk.catalog.ai_knowledge.declarative_model import ( + CatalogDeclarativeAIKnowledge, + CatalogDeclarativeKnowledgeDocument, + CatalogDeclarativeKnowledgeExtraction, + CatalogDeclarativeKnowledgeField, + CatalogDeclarativeKnowledgeFolder, +) +from gooddata_sdk.catalog.ai_knowledge.service import AIKnowledgeApiError, CatalogAIKnowledgeService + +PART = 8 + + +class FakeAIKnowledge: + """The organization AI Knowledge API, in memory, as far as the service uses it.""" + + def __init__(self) -> None: + self.folders: dict[str, dict[str, Any]] = {} + self.documents: dict[str, dict[str, Any]] = {} + self.uploads: dict[str, dict[str, Any]] = {} + self.imports: list[str] = [] + self.calls: list[str] = [] + + def __call__(self, method: str, endpoint: str, **kwargs: Any) -> requests.Response: + path = endpoint.removeprefix("/api/v1/ai/organization/knowledge") + self.calls.append(f"{method} {path}") + body = kwargs.get("json") + if method == "GET" and path == "/folders": + return _response(200, {"folders": [self._out(f) for f in self.folders.values()]}) + if method == "POST" and path == "/folders": + if any( + f["name"] == body["name"] and f.get("parentId") == body.get("parentId") for f in self.folders.values() + ): + return _response(409, {"detail": "exists"}) + folder = {"id": str(uuid.uuid4()), **body} + self.folders[folder["id"]] = folder + return _response(201, self._out(folder)) + if m := re.fullmatch(r"/folders/([^/]+)", path): + if method == "PATCH": + self.folders[m[1]].update(body) + return _response(200, self._out(self.folders[m[1]])) + if method == "DELETE": + self.folders.pop(m[1]) + return _response(200, {"success": True}) + if method == "GET" and path == "/documents": + return _response(200, {"documents": list(self.documents.values())}) + if m := re.fullmatch(r"/documents/([^/]+)", path): + self.documents[m[1]].update(body) + return _response(200, self.documents[m[1]]) + if method == "POST" and path == "/uploads": + upload_id = uuid.uuid4().hex + self.uploads[upload_id] = {**body, "parts": {}} + return _response( + 201, {"uploadId": upload_id, "partSize": PART, "partCount": max(1, -(-body["size"] // PART))} + ) + if m := re.fullmatch(r"/uploads/([^/]+)/parts/(\d+)", path): + self.uploads[m[1]]["parts"][int(m[2])] = kwargs["data"] + return _response(204, None) + if m := re.fullmatch(r"/uploads/([^/]+)/complete", path): + upload = self.uploads.pop(m[1]) + content = b"".join(upload["parts"][n] for n in sorted(upload["parts"])) + return _response( + 200, self._document(upload["filename"], upload.get("folderId"), upload.get("title"), content) + ) + if method == "POST" and path == "/imports": + self.imports.append(body["url"]) + return _response(202, self._document(body["filename"], body.get("folderId"), body.get("title"), b"")) + raise AssertionError(f"unexpected {method} {path}") + + def _document(self, filename: str, folder_id: str | None, title: str | None, content: bytes) -> dict[str, Any]: + existing = next((d for d in self.documents.values() if d["filename"] == filename), None) + document = existing or {"id": str(uuid.uuid4()), "filename": filename} + document.update({"folderId": folder_id, "title": title, "content": content.decode()}) + self.documents[document["id"]] = document + return document + + def _out(self, folder: dict[str, Any]) -> dict[str, Any]: + path, current = [], folder + while current is not None: + path.insert(0, current["name"]) + current = self.folders.get(current.get("parentId") or "") + return {"extraction": {}, "access": None, **folder, "path": path} + + +def _response(status: int, body: Any) -> requests.Response: + response = requests.Response() + response.status_code = status + response._content = json.dumps(body).encode() if body is not None else b"" + response.request = requests.Request("GET", "http://test").prepare() + return response + + +@pytest.fixture +def api() -> FakeAIKnowledge: + return FakeAIKnowledge() + + +@pytest.fixture +def service(api: FakeAIKnowledge) -> CatalogAIKnowledgeService: + client = MagicMock() + client.request.side_effect = api + return CatalogAIKnowledgeService(client) + + +def _layout(tmp_path: Path) -> CatalogDeclarativeAIKnowledge: + contracts = tmp_path / "ai_knowledge" / "Contracts" + (contracts / "files").mkdir(parents=True) + (contracts / "files" / "acme.md").write_text("ACME renews every January and pays net 30.") + (contracts / "Customers").mkdir() + (contracts / "folder.yaml").write_text( + yaml.safe_dump( + { + "name": "Contracts", + "extraction": { + "model": "gpt-4.1-mini", + "summary_instruction": "Commercial terms in three sentences.", + "fields": [{"name": "Renewal", "instruction": "When does it renew?"}], + }, + "access": {"workspaces": ["sales"]}, + } + ) + ) + (contracts / "documents.yaml").write_text( + yaml.safe_dump( + [ + {"file": "acme.md", "title": "ACME contract", "path": "files/acme.md"}, + {"file": "globex.pdf", "source": "https://files.example.com/globex.pdf"}, + ] + ) + ) + (contracts / "Customers" / "folder.yaml").write_text(yaml.safe_dump({"name": "Customers"})) + return CatalogDeclarativeAIKnowledge.load_from_disk(tmp_path) + + +def test_a_subfolder_inherits_unless_it_says_otherwise(tmp_path: Path) -> None: + layout = _layout(tmp_path) + + contracts = layout.folders[0] + customers = contracts.folders[0] + assert contracts.extraction is not None + assert contracts.extraction.fields == [ + CatalogDeclarativeKnowledgeField(name="Renewal", instruction="When does it renew?") + ] + assert contracts.access == ["sales"] + assert (customers.inherits_extraction, customers.inherits_access) == (True, True) + assert [path for path, _ in layout.walk()] == [("Contracts",), ("Contracts", "Customers")] + + +def test_a_top_level_folder_cannot_inherit(tmp_path: Path) -> None: + folder = tmp_path / "ai_knowledge" / "Loose" + folder.mkdir(parents=True) + (folder / "folder.yaml").write_text(yaml.safe_dump({"name": "Loose", "extraction": "inherit"})) + with pytest.raises(ValueError, match="top-level"): + CatalogDeclarativeAIKnowledge.load_from_disk(tmp_path) + + +def test_put_creates_the_tree_and_sends_the_documents( + service: CatalogAIKnowledgeService, api: FakeAIKnowledge, tmp_path: Path +) -> None: + result = service.put_declarative_ai_knowledge(_layout(tmp_path)) + + assert result.folders_created == ["Contracts", "Contracts/Customers"] + contracts = next(f for f in api.folders.values() if f["name"] == "Contracts") + customers = next(f for f in api.folders.values() if f["name"] == "Customers") + assert contracts["extraction"]["model"] == "gpt-4.1-mini" + assert contracts["access"] == {"workspaceIds": ["sales"]} + assert (customers["parentId"], customers["inheritExtraction"], customers["inheritAccess"]) == ( + contracts["id"], + True, + True, + ) + assert "access" not in customers and "extraction" not in customers + assert result.documents_uploaded == ["acme.md"] + assert result.documents_imported == ["globex.pdf"] + acme = next(d for d in api.documents.values() if d["filename"] == "acme.md") + assert (acme["content"], acme["folderId"], acme["title"]) == ( + "ACME renews every January and pays net 30.", + contracts["id"], + "ACME contract", + ) + assert api.imports == ["https://files.example.com/globex.pdf"] + + +def test_put_again_changes_nothing_and_moves_a_document( + service: CatalogAIKnowledgeService, api: FakeAIKnowledge, tmp_path: Path +) -> None: + layout = _layout(tmp_path) + service.put_declarative_ai_knowledge(layout) + api.calls.clear() + + again = service.put_declarative_ai_knowledge(layout) + assert (again.folders_created, again.folders_updated, again.documents_uploaded, again.documents_imported) == ( + [], + [], + [], + [], + ) + assert not [c for c in api.calls if c.startswith(("POST", "PATCH", "PUT"))] + + customers = layout.folders[0].folders[0] + acme = layout.folders[0].documents.pop(0) + customers.documents.append(CatalogDeclarativeKnowledgeDocument(file=acme.file)) + moved = service.put_declarative_ai_knowledge(layout) + assert moved.documents_moved == ["acme.md"] + + +def test_store_writes_the_tree_and_keeps_local_paths(service: CatalogAIKnowledgeService, tmp_path: Path) -> None: + layout = _layout(tmp_path) + service.put_declarative_ai_knowledge(layout) + + service.get_declarative_ai_knowledge().store_to_disk(tmp_path) + + contracts = tmp_path / "ai_knowledge" / "Contracts" + assert yaml.safe_load((contracts / "folder.yaml").read_text())["access"] == {"workspaces": ["sales"]} + assert yaml.safe_load((contracts / "Customers" / "folder.yaml").read_text()) == { + "name": "Customers", + "extraction": "inherit", + "access": "inherit", + } + documents = yaml.safe_load((contracts / "documents.yaml").read_text()) + assert {"file": "acme.md", "title": "ACME contract", "path": "files/acme.md"} in documents + assert (contracts / "files" / "acme.md").is_file() + assert CatalogDeclarativeAIKnowledge.load_from_disk(tmp_path).walk()[1][0] == ("Contracts", "Customers") + + +def test_a_document_without_file_or_source_is_reported(service: CatalogAIKnowledgeService, tmp_path: Path) -> None: + layout = CatalogDeclarativeAIKnowledge( + folders=[ + CatalogDeclarativeKnowledgeFolder( + name="Notes", + extraction=CatalogDeclarativeKnowledgeExtraction(), + documents=[CatalogDeclarativeKnowledgeDocument(file="gone.md")], + ) + ] + ) + assert service.put_declarative_ai_knowledge(layout).documents_missing == ["gone.md"] + + +def test_delete_missing_removes_folders_not_in_the_layout( + service: CatalogAIKnowledgeService, api: FakeAIKnowledge, tmp_path: Path +) -> None: + layout = _layout(tmp_path) + service.put_declarative_ai_knowledge(layout) + layout.folders[0].folders.clear() + + assert service.put_declarative_ai_knowledge(layout).folders_deleted == [] + assert service.put_declarative_ai_knowledge(layout, delete_missing=True).folders_deleted == ["Contracts/Customers"] + assert [f["name"] for f in api.folders.values()] == ["Contracts"] + + +def test_a_large_file_goes_in_parts(service: CatalogAIKnowledgeService, api: FakeAIKnowledge, tmp_path: Path) -> None: + video = tmp_path / "call.mp4" + video.write_bytes(b"0123456789abcdefXYZ") + progress: list[int] = [] + + service.upload_document(video, on_progress=lambda sent, total: progress.append(sent)) + + assert len([c for c in api.calls if "/parts/" in c]) == 3 + assert next(iter(api.documents.values()))["content"] == "0123456789abcdefXYZ" + assert max(progress) == 19 + + +def test_an_api_error_says_what_failed() -> None: + client = MagicMock() + client.request.return_value = _response(409, {"detail": "Folder 'A' already exists here"}) + with pytest.raises(AIKnowledgeApiError, match="409: Folder 'A' already exists here"): + CatalogAIKnowledgeService(client).list_folders()