diff --git a/generate_data_types.py b/generate_data_types.py new file mode 100644 index 0000000..c04eb49 --- /dev/null +++ b/generate_data_types.py @@ -0,0 +1,83 @@ +"""Generate the TypeScript declarations and the values of the Literal types of data_types.py + +The values (data_types.json) are the enums the JSON schemas of the microservices use for these +types, see generate_schemas.js +""" + +import ast +import json +from pathlib import Path + +DATA_TYPES_FILE = ( + Path(__file__).parent / "src" / "opengeodeweb_microservice" / "database" / "data_types.py" +) +OUTPUT_FILE = DATA_TYPES_FILE.with_suffix(".d.ts") +VALUES_FILE = DATA_TYPES_FILE.with_suffix(".json") +MAX_LINE_LENGTH = 100 + + +def _to_typescript(node: ast.expr, aliases: set[str]) -> list[str] | None: + """Members of the union a Literal type expression is made of, None if it is not one""" + if isinstance(node, ast.BinOp) and isinstance(node.op, ast.BitOr): + left = _to_typescript(node.left, aliases) + right = _to_typescript(node.right, aliases) + return None if left is None or right is None else left + right + if isinstance(node, ast.Name): + return [node.id] if node.id in aliases else None + if ( + isinstance(node, ast.Subscript) + and isinstance(node.value, ast.Name) + and node.value.id == "Literal" + ): + values = node.slice.elts if isinstance(node.slice, ast.Tuple) else [node.slice] + strings = [ + value.value + for value in values + if isinstance(value, ast.Constant) and isinstance(value.value, str) + ] + if len(strings) == len(values): + return [f'"{string}"' for string in strings] + return None + + +def _declaration(name: str, members: list[str]) -> str: + """Formatted as oxfmt does: on one line when it fits""" + one_line = f"export type {name} = {' | '.join(members)};\n" + if len(one_line) <= MAX_LINE_LENGTH + 1: + return one_line + union = "\n".join(f" | {member}" for member in members) + return f"export type {name} =\n{union};\n" + + +def generate() -> tuple[str, dict[str, list[str]]]: + """The TypeScript declarations, and the values of each type (its aliases expanded)""" + tree = ast.parse(DATA_TYPES_FILE.read_text()) + aliases: set[str] = set() + declarations = [] + values: dict[str, list[str]] = {} + for statement in tree.body: + if not ( + isinstance(statement, ast.Assign) + and len(statement.targets) == 1 + and isinstance(statement.targets[0], ast.Name) + ): + continue + members = _to_typescript(statement.value, aliases) + if members is None: + continue + name = statement.targets[0].id + aliases.add(name) + declarations.append(_declaration(name, members)) + values[name] = [ + value + for member in members + for value in (values[member] if member in values else [json.loads(member)]) + ] + header = f"// Generated by {Path(__file__).name} from {DATA_TYPES_FILE.name}: do not edit\n\n" + return header + "\n".join(declarations), values + + +if __name__ == "__main__": + typescript, type_values = generate() + OUTPUT_FILE.write_text(typescript) + VALUES_FILE.write_text(json.dumps(type_values, indent=2) + "\n") diff --git a/generate_schema_types.js b/generate_schema_types.js new file mode 100644 index 0000000..fc6074d --- /dev/null +++ b/generate_schema_types.js @@ -0,0 +1,293 @@ +// Generates the Python dataclasses and the TypeScript declarations of the object types of the schema_types.json of a +// Package, imported by the code generated for the routes of the microservices (see schema_types.js) + +// Node imports +import fs from "node:fs"; +import process from "node:process"; +import { fileURLToPath } from "node:url"; + +// Third party imports +import { FetchingJSONSchemaStore, InputData, JSONSchemaInput, quicktype } from "quicktype-core"; + +// Local imports +import { inlineFieldComments, loadSchemaTypes } from "./schema_types.js"; + +const HEADER = "Generated by generate_schema_types.js from schema_types.json: do not edit"; +const PYTHON_CLASS_PATTERN = /@dataclass\nclass (\w+)(?:\s*\([^)]*\))?\s*:/g; +const PYTHON_CLASS_REPLACEMENT = + "@dataclass\nclass $1(DataClassJsonMixin):\n def __post_init__(self) -> None:\n print_dataclass(self)\n"; +const PRINT_DATACLASS_IMPORT = "from opengeodeweb_microservice.schemas import print_dataclass"; +const STANDARD_MODULES = new Set(["dataclasses", "enum", "typing"]); + +async function generate(names, schemaTypes, lang, rendererOptions) { + // A package may only define aliases + if (names.length === 0) { + return ""; + } + const schemaInput = new JSONSchemaInput(new FetchingJSONSchemaStore()); + for (const name of names) { + const schema = schemaTypes.aliasPlaceholders(schemaTypes.resolvedDefinitions[name]); + await schemaInput.addSource({ name, schema: JSON.stringify(schema) }); + } + const inputData = new InputData(); + inputData.addInput(schemaInput); + const { lines } = await quicktype({ inputData, lang, rendererOptions }); + return lines.join("\n"); +} + +function withoutFields(body, names) { + return body + .split("\n") + .filter((line) => !names.has(/^ {4}(\w+):/.exec(line)?.[1])) + .join("\n"); +} + +// Quicktype declares every type flat: a type extending another one only keeps its own fields +function useInheritance(content, names, schemaTypes, blockPattern, rewrite) { + const blocks = new Map( + [...content.matchAll(blockPattern)].map(([block, name, body]) => [name, { block, body }]), + ); + let result = content; + for (const name of names) { + const baseName = schemaTypes.baseTypeNames[name]; + if (baseName === undefined) { + continue; + } + if (!names.includes(baseName)) { + throw new Error(`${name} extends ${baseName}, which is not defined by the same package`); + } + const { block, body } = blocks.get(name); + const baseFields = new Set(Object.keys(schemaTypes.resolvedDefinitions[baseName].properties)); + result = result.replace(block, rewrite(name, baseName, withoutFields(body, baseFields))); + } + return result; +} + +function assertGenerated(content, names, pattern) { + for (const name of names) { + if (!pattern(name).test(content)) { + throw new Error(`quicktype did not generate expected type ${name}`); + } + } +} + +function moduleOf(importLine) { + return /^from (\S+) import/.exec(importLine)[1]; +} + +// Ruff's isort-style order: SCREAMING_CASE names first +function compareAllNames(nameA, nameB) { + const upperA = nameA === nameA.toUpperCase(); + const upperB = nameB === nameB.toUpperCase(); + if (upperA !== upperB) { + return upperA ? -1 : 1; + } + return nameA < nameB ? -1 : 1; +} + +const PYTHON_SCALARS = { boolean: "bool", integer: "int", number: "float", string: "str" }; +const TYPESCRIPT_SCALARS = { + boolean: "boolean", + integer: "number", + number: "number", + string: "string", +}; + +// The type a non object type (e.g. DataId) is an alias of, in the language of the scalars. The other aliases of the +// Package it uses keep their name (BlockIds = list[ViewerId]), the other types are resolved +function aliasedType(definition, context) { + const { arrayOf, names, resolvedDefinitions, scalars } = context; + if (definition.title !== undefined) { + return names.includes(definition.title) + ? definition.title + : aliasedType(resolvedDefinitions[definition.title], context); + } + if (definition.type === "array") { + return arrayOf(aliasedType(definition.items, context)); + } + if (!(definition.type in scalars)) { + throw new Error(`No alias for a schema type of type ${definition.type}`); + } + return scalars[definition.type]; +} + +// The aliases of a package an alias uses +function usedAliases(definition, names) { + if (definition.title !== undefined) { + return names.includes(definition.title) ? [definition.title] : []; + } + return definition.type === "array" ? usedAliases(definition.items, names) : []; +} + +// The aliases of the non object types of a package (e.g. DataId = str), declared before its classes, each one after +// The aliases it uses +function aliases(names, schemaTypes, declaration, scalars, arrayOf) { + const { types } = schemaTypes.ownLayer; + const context = { arrayOf, names, resolvedDefinitions: schemaTypes.resolvedDefinitions, scalars }; + const ordered = []; + function declare(name) { + if (ordered.includes(name)) { + return; + } + for (const used of usedAliases(types[name], names)) { + declare(used); + } + ordered.push(name); + } + for (const name of names) { + declare(name); + } + return ordered.map((name) => declaration(name, aliasedType(types[name], context))).join("\n"); +} + +const MAX_LINE_LENGTH = 100; + +// An import line as ruff formats it: one name per line when it is too long +function formattedImport(line) { + if (line.length <= MAX_LINE_LENGTH) { + return line; + } + const [, module, names] = /^from (\S+) import (.*)$/.exec(line); + const lines = names.split(", ").map((name) => ` ${name},`); + return [`from ${module} import (`, ...lines, ")"].join("\n"); +} + +// A module of the package: written the way its linter wants it (builtin generics, sorted imports and __all__) +function pythonModule(classes, aliasDeclarations, sharedImports, layer, names) { + const typingNames = (/^from typing import (.*)$/m.exec(classes)?.[1] ?? "") + .split(", ") + .filter((name) => name && name !== "List"); + const body = classes + .replaceAll(/^(?:from|import) .*\n/gm, "") + .replaceAll(/\bList\[/g, "list[") + .trim(); + // The imports of the dataclasses, when the package defines some + const hasClasses = body.length > 0; + const standardImports = [ + ...(hasClasses ? ["from dataclasses import dataclass"] : []), + ...(/\(Enum\):/.test(classes) ? ["from enum import Enum"] : []), + ...(typingNames.length > 0 ? [`from typing import ${typingNames.join(", ")}`] : []), + ]; + // The modules of the package itself are first party, the other ones third party + const isFirstParty = (line) => moduleOf(line).startsWith(`${layer.pythonPackage}.`); + const otherImports = [...(hasClasses ? [PRINT_DATACLASS_IMPORT] : []), ...sharedImports]; + const thirdPartyImports = [ + ...(hasClasses ? ["from dataclasses_json import DataClassJsonMixin"] : []), + ...otherImports.filter((line) => !isFirstParty(line)), + ].toSorted((lineA, lineB) => (moduleOf(lineA) < moduleOf(lineB) ? -1 : 1)); + const firstPartyImports = otherImports + .filter((line) => isFirstParty(line)) + .toSorted((lineA, lineB) => (moduleOf(lineA) < moduleOf(lineB) ? -1 : 1)); + const importBlocks = [standardImports, thirdPartyImports, firstPartyImports] + .filter((block) => block.length > 0) + .map((block) => block.map((line) => formattedImport(line)).join("\n")); + // Dataclasses_json resolves the annotations at runtime: the imported shared types must stay runtime imports + // (TC001 for the first party ones, TC002 for the third party ones) + const rules = [ + ...(sharedImports.some((line) => isFirstParty(line)) ? ["TC001"] : []), + ...(sharedImports.some((line) => !isFirstParty(line)) ? ["TC002"] : []), + ]; + const noqa = rules.length > 0 ? [`# ruff: noqa: ${rules.join(", ")}`, ""] : []; + // The imports and the aliases, separated by a blank line, then the classes after two + const head = [ + ...(importBlocks.length > 0 ? [importBlocks.join("\n\n")] : []), + ...(aliasDeclarations ? [aliasDeclarations] : []), + ].join("\n\n"); + return [ + `"""${HEADER}"""`, + "", + ...noqa, + [head, ...(hasClasses ? [body] : [])].filter(Boolean).join("\n\n\n"), + "", + "", + "__all__ = [", + ...names.toSorted(compareAllNames).map((name) => ` ${JSON.stringify(name)},`), + "]", + "", + ].join("\n"); +} + +// Generates the code of the object types of the package of schemaTypes, using the types of its dependencies +async function generateSchemaTypes(schemaTypes) { + const layer = schemaTypes.ownLayer; + const names = schemaTypes.objectTypeNames(layer); + const aliasNames = Object.keys(layer.types).filter((name) => !names.includes(name)); + const dependencyLayers = schemaTypes.layers.filter((other) => other !== layer); + const { useSharedPythonTypes, useSharedTypescriptTypes } = schemaTypes.sharedTypes( + dependencyLayers, + layer, + ); + + const python = ( + await generate(names, schemaTypes, "python", { + "just-types": true, + "python-version": "3.7", + }) + ).replace(PYTHON_CLASS_PATTERN, PYTHON_CLASS_REPLACEMENT); + assertGenerated(python, names, (name) => new RegExp(`^class ${name}\\(`, "m")); + const pythonContent = useSharedPythonTypes( + useInheritance( + python, + names, + schemaTypes, + /^@dataclass\nclass (\w+)\(DataClassJsonMixin\):\n {4}def __post_init__\(self\) -> None:\n {8}print_dataclass\(self\)\n\n((?: {4}.*\n)*)/gm, + (name, baseName, ownBody) => `@dataclass\nclass ${name}(${baseName}):\n${ownBody}`, + ), + ); + // Quicktype only imports from the standard library: the other imports are the shared types + const sharedImports = [...pythonContent.matchAll(/^from (\S+) import .*$/gm)] + .filter(([, module]) => !STANDARD_MODULES.has(module)) + .map(([line]) => line); + const pythonAliases = aliases( + aliasNames, + schemaTypes, + (name, type) => `${name} = ${type}`, + PYTHON_SCALARS, + (type) => `list[${type}]`, + ); + fs.writeFileSync( + layer.pythonFile, + pythonModule(pythonContent, pythonAliases, sharedImports, layer, [...aliasNames, ...names]), + ); + + const typescript = await generate(names, schemaTypes, "typescript", { + "just-types": true, + "prefer-unions": true, + }); + assertGenerated(typescript, names, (name) => new RegExp(`^export interface ${name}\\b`, "m")); + const interfaces = inlineFieldComments( + useSharedTypescriptTypes( + useInheritance( + typescript, + names, + schemaTypes, + /^export interface (\w+) \{\n([^}]*)\}/gm, + (name, baseName, ownBody) => `export interface ${name} extends ${baseName} {\n${ownBody}}`, + ), + ), + ); + const typescriptAliases = aliases( + aliasNames, + schemaTypes, + (name, type) => `export type ${name} = ${type};`, + TYPESCRIPT_SCALARS, + (type) => `${type}[]`, + ); + const declarations = [typescriptAliases, interfaces.trim()].filter(Boolean).join("\n\n"); + fs.writeFileSync(layer.typescriptFile, `// ${HEADER}\n\n${declarations}\n`); + console.log( + `Generated ${[...aliasNames, ...names].join(", ")} in ${layer.pythonFile} and ${layer.typescriptFile}`, + ); +} + +export { generateSchemaTypes }; + +// Run directly: generates the types of the package of the current directory +if (process.argv[1] === fileURLToPath(import.meta.url)) { + try { + await generateSchemaTypes(loadSchemaTypes(process.cwd())); + } catch (error) { + console.error(error); + process.exit(1); + } +} diff --git a/generate_schemas.js b/generate_schemas.js index 952b89b..d826fa2 100755 --- a/generate_schemas.js +++ b/generate_schemas.js @@ -22,6 +22,10 @@ import { } from "quicktype-core"; import { glob } from "glob"; +// Local imports +import { generateSchemaTypes } from "./generate_schema_types.js"; +import { inlineFieldComments, loadSchemaTypes } from "./schema_types.js"; + const projectName = path.basename(process.cwd()).toLowerCase().replaceAll("-", "_"); const args = parseArgs({ @@ -61,6 +65,17 @@ const errorSchemaFile = path.join( // Route schema key holding the JSON schema of the success response body const RESPONSE_KEY = "response"; +// The schema types of the project and of its dependencies (see schema_types.js) +const schemaTypes = loadSchemaTypes(process.cwd()); +// The shared types, as quicktype names their classes (e.g. DataID for DataId) +const sharedClassNames = new Set( + [...schemaTypes.sharedTypeNames].flatMap((name) => [name, quicktypeName(name)]), +); +// Replacement of the generated declarations of the schema types, set once the project generated its own ones +let sharedTypes; +// Whether a schema of the project uses a shared type: its Python package then needs their definitions +let usedSchemaTypes = false; + // `"response": {"format": "binary"}`: the route streams a file instead of JSON function isBinaryResponse(jsonData) { return jsonData[RESPONSE_KEY]?.format === "binary"; @@ -138,7 +153,10 @@ async function appendPythonResponse(filename, jsonData, requestContent) { .replace(PYTHON_CLASS_PATTERN, PYTHON_CLASS_REPLACEMENT); // Both quicktype runs name their nested classes independently, they must not shadow each other in the module const requestClasses = new Set(exportedNames); - const duplicates = pythonClassNames(responseContent).filter((name) => requestClasses.has(name)); + // The shared types are imported, not declared by the module + const duplicates = pythonClassNames(responseContent).filter( + (name) => requestClasses.has(name) && !sharedClassNames.has(name), + ); if (duplicates.length > 0) { throw new Error( `${filename}: request and response both define class(es) ${duplicates.join(", ")}, give them distinct "title"s`, @@ -191,7 +209,10 @@ async function generatePythonFile(folderPath, filename, jsonData, fileContent) { if (RESPONSE_KEY in jsonData) { pythonContent = await appendPythonResponse(filename, jsonData, pythonContent); } - fs.writeFileSync(path.join(folderPath, filename + ".py"), pythonContent); + fs.writeFileSync( + path.join(folderPath, filename + ".py"), + sharedTypes.useSharedPythonTypes(pythonContent), + ); } // Every request/response JSON schema, fed to a single quicktype run so TS type names are unique @@ -267,8 +288,16 @@ async function return_json_schema(directoryPath, folder_path, prefix) { var schemas = {}; let initContent = ""; for (const filePath of jsonFiles) { - const fileContent = fs.readFileSync(filePath, "utf8"); - var jsonData = JSON.parse(fileContent); + const sourceData = JSON.parse(fs.readFileSync(filePath, "utf8")); + // The source only names the shared types it uses, every generated file gets their definitions + if (schemaTypes.stubSchemaTypes(sourceData)) { + fs.writeFileSync(filePath, `${JSON.stringify(sourceData, null, 2)}\n`); + } + usedSchemaTypes = schemaTypes.usesSchemaTypes(sourceData) || usedSchemaTypes; + var jsonData = schemaTypes.resolveSchemaTypes(sourceData); + // What quicktype generates the code from: the aliases (e.g. DataId) stand for their own type + const typedData = schemaTypes.aliasPlaceholders(jsonData); + const typedContent = JSON.stringify(typedData); var filename = filePath.replace(/^.*[\\/]/, "").replace(/\.[^/.]+$/, ""); try { var route = jsonData[key]; @@ -289,13 +318,13 @@ async function return_json_schema(directoryPath, folder_path, prefix) { folder_path, filename, filePath, - jsonData, - fileContent, + typedData, + typedContent, ); if (generatePython) { initContent += "from ." + filename + " import *\n"; - await generatePythonFile(folder.path, filename, jsonData, fileContent); + await generatePythonFile(folder.path, filename, typedData, typedContent); } } catch (error) { if (error.fatal) { @@ -374,7 +403,12 @@ async function generateTypescript(typesTree) { "// Generated by opengeodeweb-microservice-generate, do not edit.", `import type json from "./${path.basename(outputFile)}";`, "", - tsTypes.join("\n"), + inlineFieldComments( + sharedTypes.useSharedTypescriptTypes( + tsTypes.join("\n"), + new Set(collectTypeNames(typesTree)), + ), + ), "// `__params` and `__response` only exist at type level, to infer request/response types from a schema.", "// A type alias (not an interface) keeps the implicit index signature of the JSON type it is intersected with.", "export type TypedSchema = {", @@ -402,6 +436,10 @@ async function generateTypescript(typesTree) { } async function main() { + if (schemaTypes.ownLayer !== undefined) { + await generateSchemaTypes(schemaTypes); + } + sharedTypes = schemaTypes.sharedTypes(); const finalJson = {}; const finalTypes = {}; const { schemas, types } = await return_json_schema(directoryPath, "", prefix); @@ -409,6 +447,9 @@ async function main() { finalTypes[prefix] = types; console.log("FINAL", outputFile, finalJson); fs.writeFileSync(outputFile, JSON.stringify(finalJson, null, 2)); + if (generatePython) { + schemaTypes.writeDefinitions(usedSchemaTypes); + } await generateTypescript(finalTypes); } diff --git a/package.json b/package.json index a568418..933f394 100644 --- a/package.json +++ b/package.json @@ -22,15 +22,31 @@ }, "files": [ "generate_schemas.js", - "src/opengeodeweb_microservice/error.json" + "generate_schema_types.js", + "schema_types.js", + "src/opengeodeweb_microservice/error.json", + "src/opengeodeweb_microservice/database/data_types.d.ts", + "src/opengeodeweb_microservice/database/data_types.json", + "src/opengeodeweb_microservice/schema_types.json", + "src/opengeodeweb_microservice/schema_types.d.ts", + "src/opengeodeweb_microservice/schema_types.py" ], "type": "module", "main": "generate_schemas.js", + "exports": { + ".": "./generate_schemas.js", + "./data_types": { + "types": "./src/opengeodeweb_microservice/database/data_types.d.ts" + }, + "./schema_types": { + "types": "./src/opengeodeweb_microservice/schema_types.d.ts" + } + }, "publishConfig": { "access": "public" }, "scripts": { - "json": "echo \"Error: no test specified\" && exit 0", + "json": "python3 generate_data_types.py && node generate_schema_types.js", "test": "echo \"Error: no test specified\" && exit 0", "build": "echo \"Error: no test specified\" && exit 0" }, diff --git a/schema_types.js b/schema_types.js new file mode 100644 index 0000000..8e246b3 --- /dev/null +++ b/schema_types.js @@ -0,0 +1,598 @@ +// Shared types of the JSON schemas of the microservices: the Literal types of data_types.py (as enums) and the +// Schemas of the schema_types.json of each package, defined by the lowest package using them (e.g. GeodeId here, as +// Both OpenGeodeWeb-Back and OpenGeodeWeb-Viewer use it). +// A schema titled after one of them only names it ({"title": "GeodeId"}): the generated files get its definition +// (schema_definitions.json for the Python package at runtime), and the code generated from it uses the shared type + +// Node imports +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const microserviceDir = path.dirname(fileURLToPath(import.meta.url)); +// Values of each Literal type of data_types.py (generated by generate_data_types.py) +const dataTypes = readJson( + path.join(microserviceDir, "src", "opengeodeweb_microservice", "database", "data_types.json"), +); + +const ENUM_TYPES_TS_MODULE = "@geode/opengeodeweb-microservice/data_types"; +const ENUM_TYPES_PYTHON_MODULE = "opengeodeweb_microservice.database.data_types"; +// The value of the one value enum standing for an alias in the schemas given to quicktype (see aliasPlaceholders) +const ALIAS_PLACEHOLDER = "__schema_type__"; +// Only the packages of this scope define schema types +const SCHEMA_TYPES_SCOPE = "@geode/"; + +function readJson(file) { + return JSON.parse(fs.readFileSync(file, "utf8")); +} + +function isObject(value) { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +// The directory of an installed package, looked up the way Node does: in the node_modules of a directory and its parents +function packageDirectory(name, fromDir) { + for (let dir = fromDir; ; dir = path.dirname(dir)) { + const candidate = path.join(dir, "node_modules", name); + if (fs.existsSync(path.join(candidate, "package.json"))) { + return fs.realpathSync(candidate); + } + if (path.dirname(dir) === dir) { + return undefined; + } + } +} + +// The packages of the scope installed in the node_modules of a package itself, not of its parents. The CI installs +// The dependencies of a package there, but without their "0.0.0" versions in its package.json. In the workspace, the +// Packages are installed in the node_modules of the workspace, and the package.json of each package lists them +function installedDependencies(packageDir) { + const scopeDir = path.join(packageDir, "node_modules", SCHEMA_TYPES_SCOPE); + if (!fs.existsSync(scopeDir)) { + return []; + } + return fs + .readdirSync(scopeDir) + .filter((name) => fs.existsSync(path.join(scopeDir, name, "package.json"))) + .map((name) => `${SCHEMA_TYPES_SCOPE}${name}`); +} + +// A package and its schema types: schema_types.json and the code generated from it are in its Python package +function packageLayer(packageDir) { + const { name, dependencies = {} } = readJson(path.join(packageDir, "package.json")); + const pythonPackage = name.replace(/^@[^/]+\//, "").replaceAll("-", "_"); + const jsonFile = path.join(packageDir, "src", pythonPackage, "schema_types.json"); + return { + name, + packageDir, + pythonPackage, + dependencies: [ + ...new Set([ + ...Object.keys(dependencies).filter((dependency) => + dependency.startsWith(SCHEMA_TYPES_SCOPE), + ), + ...installedDependencies(packageDir), + ]), + ], + jsonFile, + typescriptFile: jsonFile.replace(/\.json$/, ".d.ts"), + pythonFile: jsonFile.replace(/\.json$/, ".py"), + typescriptModule: `${name}/schema_types`, + pythonModule: `${pythonPackage}.schema_types`, + }; +} + +// The packages defining schema types seen by a package: its dependencies (recursively, the lowest first), then itself +function schemaTypesLayers(packageDir) { + const layers = new Map(); + function visit(dir) { + const layer = packageLayer(dir); + if (layers.has(layer.name)) { + return; + } + layers.set(layer.name, undefined); + for (const dependency of layer.dependencies) { + const dependencyDir = packageDirectory(dependency, dir); + if (dependencyDir !== undefined) { + visit(dependencyDir); + } + } + layers.delete(layer.name); + layers.set(layer.name, layer); + } + visit(fs.realpathSync(packageDir)); + return [...layers.values()] + .filter((layer) => fs.existsSync(layer.jsonFile)) + .map((layer) => ({ ...layer, types: readJson(layer.jsonFile) })); +} + +const INTERFACE_PATTERN = /^export interface (\w+) \{\n([^}]*)\}\n\n?/gm; +const UNION_PATTERN = /^export type (\w+) =([^;]*);\n\n?/gm; +// The shared interfaces may extend another one (see the "extends" of schema_types.json) +const SHARED_INTERFACE_PATTERN = /^export interface (\w+)(?: extends (\w+))? \{\n([^}]*)\}\n\n?/gm; +// The body of an Enum can hold its docstring, followed by a blank line +const PYTHON_ENUM_PATTERN = /^class (\w+)\(Enum\):\n((?:(?: {4}.*)?\n)+)/gm; +const PYTHON_DATACLASS_PATTERN = + /^@dataclass\nclass (\w+)\(DataClassJsonMixin\):\n((?:(?:[ \t].*)?\n)*)/gm; +// The shared dataclasses may extend another one (see the "extends" of schema_types.json) +const SHARED_PYTHON_DATACLASS_PATTERN = + /^@dataclass\nclass (\w+)\((\w+)\):\n((?:(?:[ \t].*)?\n)*)/gm; + +function escapeRegExp(text) { + return text.replaceAll(/[.*+?^${}()|[\]\\]/g, String.raw`\$&`); +} + +// Quicktype aligns the fields of an interface: compare them without the alignment +function interfaceKey(body) { + return body + .split("\n") + .map((line) => line.trim().replace(/:\s+/, ": ")) + .filter(Boolean) + .toSorted() + .join("\n"); +} + +function unionKey(union) { + return union + .split("|") + .map((member) => member.trim()) + .filter(Boolean) + .toSorted() + .join(" | "); +} + +// The fields of a dataclass, whatever its typing style (List or list) +function dataclassKey(body) { + return body + .split("\n") + .map((line) => /^ {4}(\w+): (.+)$/.exec(line)) + .filter((match) => match !== null) + .map(([, field, type]) => `${field}: ${type.replaceAll(/\bList\[/g, "list[")}`) + .toSorted() + .join("\n"); +} + +function enumKey(body) { + return [...body.matchAll(/= (".*")$/gm)] + .map(([, value]) => value) + .toSorted() + .join(" | "); +} + +const enumKeys = Object.entries(dataTypes).map(([name, values]) => ({ + name, + key: values + .map((value) => JSON.stringify(value)) + .toSorted() + .join(" | "), +})); + +// Quicktype aligns the types of the fields of an interface: the fields of a body, aligned again +function alignFields(body) { + const fields = [...body.matchAll(/^ {4}(\w+\??:) +/gm)].map(([, field]) => field); + const width = Math.max(...fields.map((field) => field.length)) + 1; + return body.replaceAll(/^ {4}(\w+\??:) +/gm, (_match, field) => ` ${field.padEnd(width)}`); +} + +// Quicktype documents a field with a block comment above it: a one line description goes at the end of its line +function inlineFieldComments(content) { + return content.replaceAll(/^(export interface .*\{\n)([^}]*)\}/gm, (block, header, body) => { + const inlined = body.replaceAll( + /^( +)\/\*\*\n\1 \* (.+)\n\1 \*\/\n\1(.+;)$/gm, + (_match, indent, comment, field) => `${indent}${field} // ${comment}`, + ); + return inlined === body ? block : `${header}${alignFields(inlined)}}`; + }); +} + +// The schema types seen by the package of a directory +function loadSchemaTypes(packageDir) { + const layers = schemaTypesLayers(packageDir); + const ownLayer = layers.find((layer) => layer.packageDir === fs.realpathSync(packageDir)); + + const definedBy = new Map(Object.keys(dataTypes).map((name) => [name, undefined])); + for (const layer of layers) { + for (const name of Object.keys(layer.types)) { + // Packages not depending on each other may define the same type (e.g. ViewerId, by OpenGeodeWeb-Back and + // OpenGeodeWeb-Viewer): a package depending on both sees them as one, the same definition + const other = definedBy.get(name); + if ( + other !== undefined && + JSON.stringify(other.types[name]) === JSON.stringify(layer.types[name]) + ) { + continue; + } + if (definedBy.has(name)) { + throw new Error( + `${layer.jsonFile}: ${name} is already defined differently by ${other?.name ?? "data_types.py"}`, + ); + } + definedBy.set(name, layer); + } + } + const definitions = { + ...Object.fromEntries( + Object.entries(dataTypes).map(([name, values]) => [name, { type: "string", enum: values }]), + ), + ...Object.fromEntries(layers.flatMap((layer) => Object.entries(layer.types))), + }; + + // A definition with the shared types it uses (titled schemas) replaced by theirs + function resolveDefinition(schema) { + if (Array.isArray(schema)) { + return schema.map((item) => resolveDefinition(item)); + } + if (!isObject(schema)) { + return schema; + } + if (schema.title in definitions && Object.keys(schema).length === 1) { + return { title: schema.title, ...resolveDefinition(definitions[schema.title]) }; + } + // "extends": an object type with the properties of another one, plus its own + if (schema.extends !== undefined) { + const { extends: baseName, ...own } = schema; + const base = resolveDefinition(definitions[baseName]); + const resolvedOwn = resolveDefinition(own); + return { + ...base, + ...resolvedOwn, + properties: { ...base.properties, ...resolvedOwn.properties }, + required: [...base.required, ...(resolvedOwn.required ?? [])], + }; + } + return Object.fromEntries( + Object.entries(schema).map(([key, value]) => [key, resolveDefinition(value)]), + ); + } + + const resolvedDefinitions = Object.fromEntries( + Object.entries(definitions).map(([name, definition]) => [name, resolveDefinition(definition)]), + ); + + // The type each object type extends, if any + const baseTypeNames = Object.fromEntries( + Object.entries(definitions) + .filter(([, definition]) => definition.extends !== undefined) + .map(([name, definition]) => [name, definition.extends]), + ); + + // The object types of a package, generated as dataclasses and interfaces + function objectTypeNames(layer) { + return Object.keys(layer.types).filter((name) => resolvedDefinitions[name].type === "object"); + } + + // A schema titled after a shared type only holds its title, and its own description if the type has none + function stubSchema(schema) { + const definition = resolvedDefinitions[schema.title]; + const description = + schema.description !== undefined && definition.description === undefined + ? { description: schema.description } + : {}; + return { title: schema.title, ...description }; + } + + // Reduces every schema titled after a shared type to its stub. Returns whether the schema changed + function stubSchemaTypes(schema) { + if (Array.isArray(schema)) { + return schema.map((item) => stubSchemaTypes(item)).some(Boolean); + } + if (!isObject(schema)) { + return false; + } + if (schema.title in resolvedDefinitions) { + const stub = stubSchema(schema); + if (JSON.stringify(schema) === JSON.stringify(stub)) { + return false; + } + for (const key of Object.keys(schema)) { + delete schema[key]; + } + Object.assign(schema, stub); + return true; + } + let changed = false; + for (const value of Object.values(schema)) { + changed = stubSchemaTypes(value) || changed; + } + return changed; + } + + // A copy of a schema with the definition of every shared type it uses (the same expansion as the + // Resolve_schema_types of opengeodeweb_microservice.schemas, at runtime) + function resolveSchemaTypes(schema) { + if (Array.isArray(schema)) { + return schema.map((item) => resolveSchemaTypes(item)); + } + if (!isObject(schema)) { + return schema; + } + if (schema.title in resolvedDefinitions) { + return { ...schema, ...resolvedDefinitions[schema.title] }; + } + return Object.fromEntries( + Object.entries(schema).map(([key, value]) => [key, resolveSchemaTypes(value)]), + ); + } + + // Whether a schema uses a shared type + function usesSchemaTypes(schema) { + if (Array.isArray(schema)) { + return schema.some((item) => usesSchemaTypes(item)); + } + if (!isObject(schema)) { + return false; + } + return schema.title in resolvedDefinitions || Object.values(schema).some(usesSchemaTypes); + } + + // The definitions of the shared types, read by the Python package at runtime to resolve its schemas + const project = packageLayer(fs.realpathSync(packageDir)); + const definitionsFile = path.join( + project.packageDir, + "src", + project.pythonPackage, + "schema_definitions.json", + ); + function writeDefinitions(used) { + if (!used) { + fs.rmSync(definitionsFile, { force: true }); + return; + } + const sorted = Object.fromEntries( + Object.keys(resolvedDefinitions) + .toSorted() + .map((name) => [name, resolvedDefinitions[name]]), + ); + fs.writeFileSync(definitionsFile, `${JSON.stringify(sorted, null, 2)}\n`); + } + + // The key of each declaration of the generated code of packages, with the fields it inherits from the type it extends + function inheritedKeys(sourceLayers, fileOf, pattern, keyOf) { + const bodies = new Map(); + for (const layer of sourceLayers) { + for (const [, name, , body] of fs.readFileSync(fileOf(layer), "utf8").matchAll(pattern)) { + bodies.set(name, { body, layer }); + } + } + function fullBody(name) { + const base = baseTypeNames[name]; + const { body } = bodies.get(name); + return base === undefined ? body : `${fullBody(base)}\n${body}`; + } + return [...bodies].map(([name, { layer }]) => ({ name, layer, key: keyOf(fullBody(name)) })); + } + + // The non object types (e.g. DataId), generated as aliases: the generated code names them instead of their type + const aliasLayers = new Map( + layers.flatMap((layer) => + Object.keys(layer.types) + .filter((name) => resolvedDefinitions[name].type !== "object") + .map((name) => [name, layer]), + ), + ); + + // A copy of a resolved schema for quicktype, where each schema titled after an alias is a one value enum: quicktype + // Declares a type for it, replaced by the alias (see sharedTypes) + function aliasPlaceholders(schema) { + if (Array.isArray(schema)) { + return schema.map((item) => aliasPlaceholders(item)); + } + if (!isObject(schema)) { + return schema; + } + if (aliasLayers.has(schema.title)) { + const description = + schema.description === undefined ? {} : { description: schema.description }; + // Its own title: quicktype names its type after it, never like a type of the module (e.g. the Visibility params) + return { + title: `SchemaType${schema.title}`, + ...description, + type: "string", + enum: [`${ALIAS_PLACEHOLDER}${schema.title}`], + }; + } + return Object.fromEntries( + Object.entries(schema).map(([key, value]) => [key, aliasPlaceholders(value)]), + ); + } + + // Replacement of the declarations matching the shared types of some packages (by default all of them) by imports. + // The aliases of localLayer, whose code is being generated, are only renamed + function sharedTypes(sourceLayers = layers, localLayer = undefined) { + function aliasKeys(moduleOf) { + return [...aliasLayers] + .filter(([, layer]) => sourceLayers.includes(layer) || layer === localLayer) + .map(([name, layer]) => [ + JSON.stringify(`${ALIAS_PLACEHOLDER}${name}`), + { name, module: layer === localLayer ? undefined : moduleOf(layer) }, + ]); + } + // The shared TypeScript types, by the key of their declaration + const typescriptAliasKeys = new Map(aliasKeys((layer) => layer.typescriptModule)); + const typescriptKeys = new Map([ + ...enumKeys.map(({ name }) => [ + unionKey(dataTypes[name].map((value) => JSON.stringify(value)).join(" | ")), + { name, module: ENUM_TYPES_TS_MODULE }, + ]), + ...inheritedKeys( + sourceLayers, + (layer) => layer.typescriptFile, + SHARED_INTERFACE_PATTERN, + interfaceKey, + ).map(({ name, layer, key }) => [key, { name, module: layer.typescriptModule }]), + ]); + // The shared Python types: the aliases by the key of their placeholder enum, the other ones by name + const pythonAliasKeys = new Map(aliasKeys((layer) => layer.pythonModule)); + const pythonKeys = new Map([ + ...enumKeys.map(({ name, key }) => [name, { key, module: ENUM_TYPES_PYTHON_MODULE }]), + ...inheritedKeys( + sourceLayers, + (layer) => layer.pythonFile, + SHARED_PYTHON_DATACLASS_PATTERN, + dataclassKey, + ).map(({ name, layer, key }) => [name, { key, module: layer.pythonModule }]), + ]); + + // Removes the declarations of the shared types (each shared one of its declarations), renaming their uses + // The kept names stay declared, as aliases of their shared type (e.g. a response that is a Point3D) + function replaceDeclarations(content, declarations, sharedOf, used, keptNames = new Set()) { + let result = content; + const renamed = new Map(); + for (const declaration of declarations) { + const shared = sharedOf(declaration); + if (shared === undefined) { + continue; + } + used.push(shared); + if (keptNames.has(declaration.name)) { + result = result.replace( + declaration.declaration, + `export type ${declaration.name} = ${shared.name};\n\n`, + ); + } else { + renamed.set(declaration.name, shared.name); + result = result.replace(declaration.declaration, ""); + } + } + for (const [name, sharedName] of renamed) { + if (name !== sharedName) { + result = result.replaceAll(new RegExp(`\\b${escapeRegExp(name)}\\b`, "g"), sharedName); + } + } + return result; + } + + // The import lines of the used shared types, one per module + function importLines(used, line) { + const imports = Map.groupBy( + used.filter(({ module }) => module !== undefined), + ({ module }) => module, + ); + return [...imports].map(([module, types]) => + line(module, [...new Set(types.map(({ name, imported }) => imported ?? name))].toSorted()), + ); + } + + function typescriptDeclarations(content) { + return [ + ...[...content.matchAll(INTERFACE_PATTERN)].map(([declaration, name, body]) => ({ + declaration, + name, + key: interfaceKey(body), + })), + ...[...content.matchAll(UNION_PATTERN)].map(([declaration, name, union]) => ({ + declaration, + name, + key: unionKey(union), + })), + ]; + } + + // Quicktype declares its own type for each use of a shared type, named after its path when it is used by + // Several routes: replace every declaration matching a shared type by an import of it. The aliases first, as + // The shared interfaces use them. The kept names (e.g. the params and responses of the routes, referenced + // Elsewhere) stay declared as aliases of their shared type + function useSharedTypescriptTypes(content, keptNames = new Set()) { + const used = []; + const withAliases = replaceDeclarations( + content, + typescriptDeclarations(content), + ({ key }) => typescriptAliasKeys.get(key), + used, + keptNames, + ); + const result = replaceDeclarations( + withAliases, + typescriptDeclarations(withAliases), + ({ key }) => typescriptKeys.get(key), + used, + keptNames, + ); + const lines = importLines( + used, + (module, names) => `import type { ${names.join(", ")} } from "${module}";\n`, + ); + return lines.length === 0 ? result : `${lines.join("")}\n${result}`; + } + + // Quicktype declares its own Enum or dataclass for each shared type, named after its title: import the shared + // One instead when it is the same type (the enums are Literal types, so the values stay plain strings) + function useSharedPythonTypes(content) { + const used = []; + const enums = (text) => + [...text.matchAll(PYTHON_ENUM_PATTERN)].map(([declaration, name, body]) => ({ + declaration, + name, + key: enumKey(body), + })); + // A module declaring a class named after an alias (e.g. the Visibility params of visibility.py) imports it as + // VisibilityType + const classNames = new Set([...content.matchAll(/^class (\w+)\(/gm)].map(([, name]) => name)); + const withAliases = replaceDeclarations( + content, + enums(content), + ({ key }) => { + const shared = pythonAliasKeys.get(key); + if (shared === undefined || !classNames.has(shared.name)) { + return shared; + } + const renamed = `${shared.name}Type`; + return { ...shared, name: renamed, imported: `${shared.name} as ${renamed}` }; + }, + used, + ); + const sameType = ({ name, key }) => { + const shared = pythonKeys.get(name); + return shared !== undefined && shared.key === key + ? { name, module: shared.module } + : undefined; + }; + let result = replaceDeclarations( + withAliases, + [ + ...enums(withAliases), + ...[...withAliases.matchAll(PYTHON_DATACLASS_PATTERN)].map( + ([declaration, name, body]) => ({ + declaration, + name, + key: dataclassKey(body), + }), + ), + ], + sameType, + used, + ); + if (used.length === 0) { + return content; + } + if (!/\(Enum\):/.test(result)) { + result = result.replace(/^from enum import Enum\n/m, ""); + } + const lines = importLines( + used, + (module, names) => `from ${module} import ${names.join(", ")}\n`, + ); + return `${lines.join("")}${result}`; + } + + return { useSharedPythonTypes, useSharedTypescriptTypes }; + } + + return { + baseTypeNames, + layers, + objectTypeNames, + ownLayer, + resolvedDefinitions, + aliasPlaceholders, + // The types the generated code imports instead of declaring them + sharedTypeNames: new Set(Object.keys(resolvedDefinitions)), + resolveSchemaTypes, + sharedTypes, + stubSchemaTypes, + usesSchemaTypes, + writeDefinitions, + }; +} + +export { inlineFieldComments, loadSchemaTypes }; diff --git a/src/opengeodeweb_microservice/database/data.py b/src/opengeodeweb_microservice/database/data.py index c89c4a6..7be0a0e 100644 --- a/src/opengeodeweb_microservice/database/data.py +++ b/src/opengeodeweb_microservice/database/data.py @@ -3,6 +3,8 @@ from sqlalchemy import String, select from sqlalchemy.orm import Mapped, mapped_column +from opengeodeweb_microservice.schema_types import DataId, GeodeId # noqa: TC001 + from .base import Base from .connection import get_session @@ -17,10 +19,10 @@ class Data(Base): __tablename__ = "datas" - id: Mapped[str] = mapped_column( + id: Mapped[DataId] = mapped_column( String, primary_key=True, default=lambda: str(uuid.uuid4()).replace("-", "") ) - geode_id: Mapped[str] = mapped_column(String, nullable=False) + geode_id: Mapped[GeodeId] = mapped_column(String, nullable=False) geode_object_type: Mapped[GeodeObjectType] = mapped_column(String, nullable=False) viewer_object_type: Mapped[ViewerObjectType] = mapped_column(String, nullable=False) viewer_elements_type: Mapped[ViewerElementsType] = mapped_column(String, nullable=False) @@ -30,7 +32,7 @@ class Data(Base): @staticmethod def create( - geode_id: str, + geode_id: GeodeId, geode_object_type: GeodeObjectType, viewer_object_type: ViewerObjectType, viewer_elements_type: ViewerElementsType, @@ -48,7 +50,7 @@ def create( return data_entry @staticmethod - def get(data_id: str) -> Data | None: + def get(data_id: DataId) -> Data | None: session = get_session() data_query = select(Data).where(Data.id == data_id) return session.scalars(data_query).first() diff --git a/src/opengeodeweb_microservice/database/data_types.d.ts b/src/opengeodeweb_microservice/database/data_types.d.ts new file mode 100644 index 0000000..5bfa194 --- /dev/null +++ b/src/opengeodeweb_microservice/database/data_types.d.ts @@ -0,0 +1,45 @@ +// Generated by generate_data_types.py from data_types.py: do not edit + +export type GeodePointMeshType = "PointSet2D" | "PointSet3D"; + +export type GeodeEdgeMeshType = "EdgedCurve2D" | "EdgedCurve3D"; + +export type GeodePolygonMeshType = + | "RasterImage2D" + | "PolygonalSurface2D" + | "PolygonalSurface3D" + | "TriangulatedSurface2D" + | "TriangulatedSurface3D" + | "RegularGrid2D" + | "LightRegularGrid2D"; + +export type GeodePolyhedronMeshType = + | "RasterImage3D" + | "PolyhedralSolid3D" + | "TetrahedralSolid3D" + | "HybridSolid3D" + | "RegularGrid3D" + | "LightRegularGrid3D"; + +export type GeodeMeshType = + | "VertexSet" + | "Graph" + | GeodePointMeshType + | GeodeEdgeMeshType + | GeodePolygonMeshType + | GeodePolyhedronMeshType; + +export type GeodeModelType = + | "BRep" + | "Section" + | "StructuralModel" + | "CrossSection" + | "ImplicitStructuralModel" + | "ImplicitCrossSection" + | "HorizonStack3D"; + +export type GeodeObjectType = GeodeMeshType | GeodeModelType; + +export type ViewerObjectType = "mesh" | "model"; + +export type ViewerElementsType = "points" | "edges" | "polygons" | "polyhedra" | "default"; diff --git a/src/opengeodeweb_microservice/database/data_types.json b/src/opengeodeweb_microservice/database/data_types.json new file mode 100644 index 0000000..4bb3ae9 --- /dev/null +++ b/src/opengeodeweb_microservice/database/data_types.json @@ -0,0 +1,96 @@ +{ + "GeodePointMeshType": [ + "PointSet2D", + "PointSet3D" + ], + "GeodeEdgeMeshType": [ + "EdgedCurve2D", + "EdgedCurve3D" + ], + "GeodePolygonMeshType": [ + "RasterImage2D", + "PolygonalSurface2D", + "PolygonalSurface3D", + "TriangulatedSurface2D", + "TriangulatedSurface3D", + "RegularGrid2D", + "LightRegularGrid2D" + ], + "GeodePolyhedronMeshType": [ + "RasterImage3D", + "PolyhedralSolid3D", + "TetrahedralSolid3D", + "HybridSolid3D", + "RegularGrid3D", + "LightRegularGrid3D" + ], + "GeodeMeshType": [ + "VertexSet", + "Graph", + "PointSet2D", + "PointSet3D", + "EdgedCurve2D", + "EdgedCurve3D", + "RasterImage2D", + "PolygonalSurface2D", + "PolygonalSurface3D", + "TriangulatedSurface2D", + "TriangulatedSurface3D", + "RegularGrid2D", + "LightRegularGrid2D", + "RasterImage3D", + "PolyhedralSolid3D", + "TetrahedralSolid3D", + "HybridSolid3D", + "RegularGrid3D", + "LightRegularGrid3D" + ], + "GeodeModelType": [ + "BRep", + "Section", + "StructuralModel", + "CrossSection", + "ImplicitStructuralModel", + "ImplicitCrossSection", + "HorizonStack3D" + ], + "GeodeObjectType": [ + "VertexSet", + "Graph", + "PointSet2D", + "PointSet3D", + "EdgedCurve2D", + "EdgedCurve3D", + "RasterImage2D", + "PolygonalSurface2D", + "PolygonalSurface3D", + "TriangulatedSurface2D", + "TriangulatedSurface3D", + "RegularGrid2D", + "LightRegularGrid2D", + "RasterImage3D", + "PolyhedralSolid3D", + "TetrahedralSolid3D", + "HybridSolid3D", + "RegularGrid3D", + "LightRegularGrid3D", + "BRep", + "Section", + "StructuralModel", + "CrossSection", + "ImplicitStructuralModel", + "ImplicitCrossSection", + "HorizonStack3D" + ], + "ViewerObjectType": [ + "mesh", + "model" + ], + "ViewerElementsType": [ + "points", + "edges", + "polygons", + "polyhedra", + "default" + ] +} diff --git a/src/opengeodeweb_microservice/schema_types.d.ts b/src/opengeodeweb_microservice/schema_types.d.ts new file mode 100644 index 0000000..0546697 --- /dev/null +++ b/src/opengeodeweb_microservice/schema_types.d.ts @@ -0,0 +1,4 @@ +// Generated by generate_schema_types.js from schema_types.json: do not edit + +export type DataId = string; +export type GeodeId = string; diff --git a/src/opengeodeweb_microservice/schema_types.json b/src/opengeodeweb_microservice/schema_types.json new file mode 100644 index 0000000..6fdfeff --- /dev/null +++ b/src/opengeodeweb_microservice/schema_types.json @@ -0,0 +1,12 @@ +{ + "DataId": { + "type": "string", + "minLength": 32, + "maxLength": 32 + }, + "GeodeId": { + "type": "string", + "minLength": 36, + "maxLength": 36 + } +} diff --git a/src/opengeodeweb_microservice/schema_types.py b/src/opengeodeweb_microservice/schema_types.py new file mode 100644 index 0000000..ecefde3 --- /dev/null +++ b/src/opengeodeweb_microservice/schema_types.py @@ -0,0 +1,10 @@ +"""Generated by generate_schema_types.js from schema_types.json: do not edit""" + +DataId = str +GeodeId = str + + +__all__ = [ + "DataId", + "GeodeId", +] diff --git a/src/opengeodeweb_microservice/schemas.py b/src/opengeodeweb_microservice/schemas.py index 5e36449..44236bf 100644 --- a/src/opengeodeweb_microservice/schemas.py +++ b/src/opengeodeweb_microservice/schemas.py @@ -1,4 +1,6 @@ +import copy import dataclasses +import functools import json import sys from collections.abc import Sized @@ -14,6 +16,8 @@ type SchemaDict = dict[str, str] ERROR_SCHEMA_PATH = Path(__file__).parent / "error.json" +# Generated in each package by generate_schemas.js, see schema_types.js +SCHEMA_DEFINITIONS_FILENAME = "schema_definitions.json" MAX_VALUE_LENGTH = 80 @@ -48,19 +52,49 @@ def print_dataclass(instance: DataclassInstance) -> None: sys.stdout.flush() +@functools.cache +def schema_definitions(directory: Path) -> dict[str, dict[str, Any]]: + """Definitions of the shared schema types for the schemas of a directory. + + Read from the schema_definitions.json generated in its package, by the closest parent directory. + """ + for parent in (directory, *directory.parents): + definitions_file = parent / SCHEMA_DEFINITIONS_FILENAME + if definitions_file.is_file(): + with definitions_file.open() as file: + definitions: dict[str, dict[str, Any]] = json.load(file) + return definitions + return {} + + +def resolve_schema_types(schema: object, definitions: dict[str, dict[str, Any]]) -> object: + """Give every schema titled after a shared type (e.g. {"title": "GeodeId"}) its definition.""" + if isinstance(schema, list): + return [resolve_schema_types(item, definitions) for item in schema] + if not isinstance(schema, dict): + return schema + title = schema.get("title") + if isinstance(title, str) and title in definitions: + return {**schema, **copy.deepcopy(definitions[title])} + return {key: resolve_schema_types(value, definitions) for key, value in schema.items()} + + +def _read_schema(json_file: Path) -> dict[str, Any]: + with json_file.open() as file: + schema = resolve_schema_types(json.load(file), schema_definitions(json_file.parent)) + if not isinstance(schema, dict): + message = f"{json_file}: a schema must be a JSON object" + raise TypeError(message) + return schema + + def get_schemas_dict(path: Path) -> dict[str, SchemaDict]: - schemas_dict: dict[str, SchemaDict] = {} - for json_file in path.glob("*.json"): - with json_file.open() as file: - schemas_dict[json_file.stem] = json.load(file) - return schemas_dict + return {json_file.stem: _read_schema(json_file) for json_file in path.glob("*.json")} def load_schema(python_file: Path) -> dict[str, Any]: """Load the JSON route schema sitting next to a generated schema module.""" - with python_file.with_suffix(".json").open() as file: - schema: dict[str, Any] = json.load(file) - return schema + return _read_schema(python_file.with_suffix(".json")) @dataclass(frozen=True) diff --git a/tests/test_schemas.py b/tests/test_schemas.py index c72759e..ab75e13 100644 --- a/tests/test_schemas.py +++ b/tests/test_schemas.py @@ -1,14 +1,21 @@ import json from dataclasses import dataclass +from typing import TYPE_CHECKING import fastjsonschema from opengeodeweb_microservice.schemas import ( ERROR_SCHEMA_PATH, + SCHEMA_DEFINITIONS_FILENAME, ErrorResponse, format_dataclass, + get_schemas_dict, + load_schema, ) +if TYPE_CHECKING: + from pathlib import Path + def test_error_response_matches_error_schema() -> None: with ERROR_SCHEMA_PATH.open() as file: @@ -35,3 +42,37 @@ def test_format_dataclass_truncates_long_values() -> None: ) assert text.startswith("_Outer(name='short', content='xxxxxxxxxxxxxxxxxxx... (str, len=1000)") assert "inner=_Inner(values=[0, 1, 2, 3, 4, 5, 6... (list, len=500))" in text + + +def test_load_schema_resolves_the_shared_types(tmp_path: Path) -> None: + geode_id = {"type": "string", "minLength": 36, "maxLength": 36} + (tmp_path / SCHEMA_DEFINITIONS_FILENAME).write_text(json.dumps({"GeodeId": geode_id})) + schemas_dir = tmp_path / "routes" / "schemas" + schemas_dir.mkdir(parents=True) + route = { + "route": "components", + "type": "object", + "properties": { + "geode_id": {"title": "GeodeId", "description": "Of the model"}, + "component_ids": {"type": "array", "items": {"title": "GeodeId"}}, + "other": {"title": "Other", "type": "string"}, + }, + } + (schemas_dir / "components.json").write_text(json.dumps(route)) + + schema = load_schema(schemas_dir / "components.py") + + properties = schema["properties"] + assert properties["geode_id"] == {"title": "GeodeId", "description": "Of the model", **geode_id} + assert properties["component_ids"]["items"] == {"title": "GeodeId", **geode_id} + assert properties["other"] == {"title": "Other", "type": "string"} + assert get_schemas_dict(schemas_dir) == {"components": schema} + validate = fastjsonschema.compile(schema) + validate({"geode_id": "0" * 36, "component_ids": [], "other": "x"}) + + +def test_load_schema_without_definitions(tmp_path: Path) -> None: + route = {"route": "name", "properties": {"name": {"title": "Name", "type": "string"}}} + (tmp_path / "name.json").write_text(json.dumps(route)) + + assert load_schema(tmp_path / "name.py") == route