Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
83 changes: 83 additions & 0 deletions generate_data_types.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
"""Generate the TypeScript declarations and the values of the Literal types of data_types.py

The values (data_types.json) are the enums the JSON schemas of the microservices use for these
types, see generate_schemas.js
"""

import ast
import json
from pathlib import Path

DATA_TYPES_FILE = (
Path(__file__).parent / "src" / "opengeodeweb_microservice" / "database" / "data_types.py"
)
OUTPUT_FILE = DATA_TYPES_FILE.with_suffix(".d.ts")
VALUES_FILE = DATA_TYPES_FILE.with_suffix(".json")
MAX_LINE_LENGTH = 100


def _to_typescript(node: ast.expr, aliases: set[str]) -> list[str] | None:
"""Members of the union a Literal type expression is made of, None if it is not one"""
if isinstance(node, ast.BinOp) and isinstance(node.op, ast.BitOr):
left = _to_typescript(node.left, aliases)
right = _to_typescript(node.right, aliases)
return None if left is None or right is None else left + right
if isinstance(node, ast.Name):
return [node.id] if node.id in aliases else None
if (
isinstance(node, ast.Subscript)
and isinstance(node.value, ast.Name)
and node.value.id == "Literal"
):
values = node.slice.elts if isinstance(node.slice, ast.Tuple) else [node.slice]
strings = [
value.value
for value in values
if isinstance(value, ast.Constant) and isinstance(value.value, str)
]
if len(strings) == len(values):
return [f'"{string}"' for string in strings]
return None


def _declaration(name: str, members: list[str]) -> str:
"""Formatted as oxfmt does: on one line when it fits"""
one_line = f"export type {name} = {' | '.join(members)};\n"
if len(one_line) <= MAX_LINE_LENGTH + 1:
return one_line
union = "\n".join(f" | {member}" for member in members)
return f"export type {name} =\n{union};\n"


def generate() -> tuple[str, dict[str, list[str]]]:
"""The TypeScript declarations, and the values of each type (its aliases expanded)"""
tree = ast.parse(DATA_TYPES_FILE.read_text())
aliases: set[str] = set()
declarations = []
values: dict[str, list[str]] = {}
for statement in tree.body:
if not (
isinstance(statement, ast.Assign)
and len(statement.targets) == 1
and isinstance(statement.targets[0], ast.Name)
):
continue
members = _to_typescript(statement.value, aliases)
if members is None:
continue
name = statement.targets[0].id
aliases.add(name)
declarations.append(_declaration(name, members))
values[name] = [
value
for member in members
for value in (values[member] if member in values else [json.loads(member)])
]
header = f"// Generated by {Path(__file__).name} from {DATA_TYPES_FILE.name}: do not edit\n\n"
return header + "\n".join(declarations), values


if __name__ == "__main__":
typescript, type_values = generate()
OUTPUT_FILE.write_text(typescript)
VALUES_FILE.write_text(json.dumps(type_values, indent=2) + "\n")
293 changes: 293 additions & 0 deletions generate_schema_types.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,293 @@
// Generates the Python dataclasses and the TypeScript declarations of the object types of the schema_types.json of a
// Package, imported by the code generated for the routes of the microservices (see schema_types.js)

// Node imports
import fs from "node:fs";
import process from "node:process";
import { fileURLToPath } from "node:url";

// Third party imports
import { FetchingJSONSchemaStore, InputData, JSONSchemaInput, quicktype } from "quicktype-core";

// Local imports
import { inlineFieldComments, loadSchemaTypes } from "./schema_types.js";

const HEADER = "Generated by generate_schema_types.js from schema_types.json: do not edit";
const PYTHON_CLASS_PATTERN = /@dataclass\nclass (\w+)(?:\s*\([^)]*\))?\s*:/g;
const PYTHON_CLASS_REPLACEMENT =
"@dataclass\nclass $1(DataClassJsonMixin):\n def __post_init__(self) -> None:\n print_dataclass(self)\n";
const PRINT_DATACLASS_IMPORT = "from opengeodeweb_microservice.schemas import print_dataclass";
const STANDARD_MODULES = new Set(["dataclasses", "enum", "typing"]);

async function generate(names, schemaTypes, lang, rendererOptions) {
// A package may only define aliases
if (names.length === 0) {
return "";
}
const schemaInput = new JSONSchemaInput(new FetchingJSONSchemaStore());
for (const name of names) {
const schema = schemaTypes.aliasPlaceholders(schemaTypes.resolvedDefinitions[name]);
await schemaInput.addSource({ name, schema: JSON.stringify(schema) });
}
const inputData = new InputData();
inputData.addInput(schemaInput);
const { lines } = await quicktype({ inputData, lang, rendererOptions });
return lines.join("\n");
}

function withoutFields(body, names) {
return body
.split("\n")
.filter((line) => !names.has(/^ {4}(\w+):/.exec(line)?.[1]))
.join("\n");
}

// Quicktype declares every type flat: a type extending another one only keeps its own fields
function useInheritance(content, names, schemaTypes, blockPattern, rewrite) {
const blocks = new Map(
[...content.matchAll(blockPattern)].map(([block, name, body]) => [name, { block, body }]),
);
let result = content;
for (const name of names) {
const baseName = schemaTypes.baseTypeNames[name];
if (baseName === undefined) {
continue;
}
if (!names.includes(baseName)) {
throw new Error(`${name} extends ${baseName}, which is not defined by the same package`);
}
const { block, body } = blocks.get(name);
const baseFields = new Set(Object.keys(schemaTypes.resolvedDefinitions[baseName].properties));
result = result.replace(block, rewrite(name, baseName, withoutFields(body, baseFields)));
}
return result;
}

function assertGenerated(content, names, pattern) {
for (const name of names) {
if (!pattern(name).test(content)) {
throw new Error(`quicktype did not generate expected type ${name}`);
}
}
}

function moduleOf(importLine) {
return /^from (\S+) import/.exec(importLine)[1];
}

// Ruff's isort-style order: SCREAMING_CASE names first
function compareAllNames(nameA, nameB) {
const upperA = nameA === nameA.toUpperCase();
const upperB = nameB === nameB.toUpperCase();
if (upperA !== upperB) {
return upperA ? -1 : 1;
}
return nameA < nameB ? -1 : 1;
}

const PYTHON_SCALARS = { boolean: "bool", integer: "int", number: "float", string: "str" };
const TYPESCRIPT_SCALARS = {
boolean: "boolean",
integer: "number",
number: "number",
string: "string",
};

// The type a non object type (e.g. DataId) is an alias of, in the language of the scalars. The other aliases of the
// Package it uses keep their name (BlockIds = list[ViewerId]), the other types are resolved
function aliasedType(definition, context) {
const { arrayOf, names, resolvedDefinitions, scalars } = context;
if (definition.title !== undefined) {
return names.includes(definition.title)
? definition.title
: aliasedType(resolvedDefinitions[definition.title], context);
}
if (definition.type === "array") {
return arrayOf(aliasedType(definition.items, context));
}
if (!(definition.type in scalars)) {
throw new Error(`No alias for a schema type of type ${definition.type}`);
}
return scalars[definition.type];
}

// The aliases of a package an alias uses
function usedAliases(definition, names) {
if (definition.title !== undefined) {
return names.includes(definition.title) ? [definition.title] : [];
}
return definition.type === "array" ? usedAliases(definition.items, names) : [];
}

// The aliases of the non object types of a package (e.g. DataId = str), declared before its classes, each one after
// The aliases it uses
function aliases(names, schemaTypes, declaration, scalars, arrayOf) {
const { types } = schemaTypes.ownLayer;
const context = { arrayOf, names, resolvedDefinitions: schemaTypes.resolvedDefinitions, scalars };
const ordered = [];
function declare(name) {
if (ordered.includes(name)) {
return;
}
for (const used of usedAliases(types[name], names)) {
declare(used);
}
ordered.push(name);
}
for (const name of names) {
declare(name);
}
return ordered.map((name) => declaration(name, aliasedType(types[name], context))).join("\n");
}

const MAX_LINE_LENGTH = 100;

// An import line as ruff formats it: one name per line when it is too long
function formattedImport(line) {
if (line.length <= MAX_LINE_LENGTH) {
return line;
}
const [, module, names] = /^from (\S+) import (.*)$/.exec(line);
const lines = names.split(", ").map((name) => ` ${name},`);
return [`from ${module} import (`, ...lines, ")"].join("\n");
}

// A module of the package: written the way its linter wants it (builtin generics, sorted imports and __all__)
function pythonModule(classes, aliasDeclarations, sharedImports, layer, names) {
const typingNames = (/^from typing import (.*)$/m.exec(classes)?.[1] ?? "")
.split(", ")
.filter((name) => name && name !== "List");
const body = classes
.replaceAll(/^(?:from|import) .*\n/gm, "")
.replaceAll(/\bList\[/g, "list[")
.trim();
// The imports of the dataclasses, when the package defines some
const hasClasses = body.length > 0;
const standardImports = [
...(hasClasses ? ["from dataclasses import dataclass"] : []),
...(/\(Enum\):/.test(classes) ? ["from enum import Enum"] : []),
...(typingNames.length > 0 ? [`from typing import ${typingNames.join(", ")}`] : []),
];
// The modules of the package itself are first party, the other ones third party
const isFirstParty = (line) => moduleOf(line).startsWith(`${layer.pythonPackage}.`);
const otherImports = [...(hasClasses ? [PRINT_DATACLASS_IMPORT] : []), ...sharedImports];
const thirdPartyImports = [
...(hasClasses ? ["from dataclasses_json import DataClassJsonMixin"] : []),
...otherImports.filter((line) => !isFirstParty(line)),
].toSorted((lineA, lineB) => (moduleOf(lineA) < moduleOf(lineB) ? -1 : 1));
const firstPartyImports = otherImports
.filter((line) => isFirstParty(line))
.toSorted((lineA, lineB) => (moduleOf(lineA) < moduleOf(lineB) ? -1 : 1));
const importBlocks = [standardImports, thirdPartyImports, firstPartyImports]
.filter((block) => block.length > 0)
.map((block) => block.map((line) => formattedImport(line)).join("\n"));
// Dataclasses_json resolves the annotations at runtime: the imported shared types must stay runtime imports
// (TC001 for the first party ones, TC002 for the third party ones)
const rules = [
...(sharedImports.some((line) => isFirstParty(line)) ? ["TC001"] : []),
...(sharedImports.some((line) => !isFirstParty(line)) ? ["TC002"] : []),
];
const noqa = rules.length > 0 ? [`# ruff: noqa: ${rules.join(", ")}`, ""] : [];
// The imports and the aliases, separated by a blank line, then the classes after two
const head = [
...(importBlocks.length > 0 ? [importBlocks.join("\n\n")] : []),
...(aliasDeclarations ? [aliasDeclarations] : []),
].join("\n\n");
return [
`"""${HEADER}"""`,
"",
...noqa,
[head, ...(hasClasses ? [body] : [])].filter(Boolean).join("\n\n\n"),
"",
"",
"__all__ = [",
...names.toSorted(compareAllNames).map((name) => ` ${JSON.stringify(name)},`),
"]",
"",
].join("\n");
}

// Generates the code of the object types of the package of schemaTypes, using the types of its dependencies
async function generateSchemaTypes(schemaTypes) {
const layer = schemaTypes.ownLayer;
const names = schemaTypes.objectTypeNames(layer);
const aliasNames = Object.keys(layer.types).filter((name) => !names.includes(name));
const dependencyLayers = schemaTypes.layers.filter((other) => other !== layer);
const { useSharedPythonTypes, useSharedTypescriptTypes } = schemaTypes.sharedTypes(
dependencyLayers,
layer,
);

const python = (
await generate(names, schemaTypes, "python", {
"just-types": true,
"python-version": "3.7",
})
).replace(PYTHON_CLASS_PATTERN, PYTHON_CLASS_REPLACEMENT);
assertGenerated(python, names, (name) => new RegExp(`^class ${name}\\(`, "m"));
const pythonContent = useSharedPythonTypes(
useInheritance(
python,
names,
schemaTypes,
/^@dataclass\nclass (\w+)\(DataClassJsonMixin\):\n {4}def __post_init__\(self\) -> None:\n {8}print_dataclass\(self\)\n\n((?: {4}.*\n)*)/gm,
(name, baseName, ownBody) => `@dataclass\nclass ${name}(${baseName}):\n${ownBody}`,
),
);
// Quicktype only imports from the standard library: the other imports are the shared types
const sharedImports = [...pythonContent.matchAll(/^from (\S+) import .*$/gm)]
.filter(([, module]) => !STANDARD_MODULES.has(module))
.map(([line]) => line);
const pythonAliases = aliases(
aliasNames,
schemaTypes,
(name, type) => `${name} = ${type}`,
PYTHON_SCALARS,
(type) => `list[${type}]`,
);
fs.writeFileSync(
layer.pythonFile,
pythonModule(pythonContent, pythonAliases, sharedImports, layer, [...aliasNames, ...names]),
);

const typescript = await generate(names, schemaTypes, "typescript", {
"just-types": true,
"prefer-unions": true,
});
assertGenerated(typescript, names, (name) => new RegExp(`^export interface ${name}\\b`, "m"));
const interfaces = inlineFieldComments(
useSharedTypescriptTypes(
useInheritance(
typescript,
names,
schemaTypes,
/^export interface (\w+) \{\n([^}]*)\}/gm,
(name, baseName, ownBody) => `export interface ${name} extends ${baseName} {\n${ownBody}}`,
),
),
);
const typescriptAliases = aliases(
aliasNames,
schemaTypes,
(name, type) => `export type ${name} = ${type};`,
TYPESCRIPT_SCALARS,
(type) => `${type}[]`,
);
const declarations = [typescriptAliases, interfaces.trim()].filter(Boolean).join("\n\n");
fs.writeFileSync(layer.typescriptFile, `// ${HEADER}\n\n${declarations}\n`);
console.log(
`Generated ${[...aliasNames, ...names].join(", ")} in ${layer.pythonFile} and ${layer.typescriptFile}`,
);
}

export { generateSchemaTypes };

// Run directly: generates the types of the package of the current directory
if (process.argv[1] === fileURLToPath(import.meta.url)) {
try {
await generateSchemaTypes(loadSchemaTypes(process.cwd()));
} catch (error) {
console.error(error);
process.exit(1);
}
}
Loading
Loading