diff --git a/MANIFEST.in b/MANIFEST.in index 8f812cd..9f9035b 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,5 @@ recursive-include test *.py include README* LICENSE distribute_setup.py CHANGES* run_tests.py +include conftest.py requirements-test.txt diff --git a/README.md b/README.md index b953229..c00ed15 100644 --- a/README.md +++ b/README.md @@ -9,9 +9,12 @@ Version Prerequisites -------------- -1. Install Python 3.8 or newer versions or Jython 2.7.x . -2. SQLAlchemy version between 0.7.3 - 2.0.x. +1. Install Python 3.10 to 3.15 or Jython 2.7.x . +2. SQLAlchemy version between 1.4.x - 2.1.x. 3. IBM_DB driver and IBM_DB_DBI wrapper 1.0.1 or higher. + +Note: Older SQLAlchemy versions are supported by IBM_DB_SA versions up to and including 0.4.4. Starting with versions after 0.4.4, IBM_DB_SA supports SQLAlchemy 1.4.x through 2.1.x. + ``` Install ibm_db driver with below commands: Linux and Windows: @@ -47,7 +50,7 @@ Please follow the steps provided to Install "Easy Install" in the link above and 4. In case you want to control the sqlalchemy version use the following commands ``` Install SQLAlchemy supported version: - pip install sqlalchemy==1.3.23 + pip install "sqlalchemy==2.1.*" pip install ibm_db_sa ``` @@ -83,6 +86,29 @@ For a local socket connection, exclude the "host" and "port" portions:: e = create_engine("ibm_db_sa://user:pass@/database") ``` +Testing +------- +IBM_DB_SA is tested with SQLAlchemy's pytest-based test suite for third-party +dialects, including the SQLAlchemy dialect compliance suite +(`sqlalchemy.testing.suite`). The tests can be run against SQLAlchemy 1.4.x, +2.0.x and 2.1.x. + +Quick start, from the root of the repository: +``` +pip install -e . +pip install -r requirements-test.txt + +# tests that do not need a database: +pytest test/test_dialect.py + +# all tests, against a Db2 database: +pytest --dburi "ibm_db_sa://user:pass@host:port/DBNAME" test/ +``` + +For full instructions (configuring the database connection, running single +tests, testing with a specific SQLAlchemy version and investigating +failures), see [README.unittests.md](README.unittests.md). + ## Logging in ibm_db_sa The `ibm_db_sa` module provides built-in logging support to help debug SQLAlchemy dialect operations, connection handling, reflection, and SQL execution. Logging can be enabled using the `ibmdbsa_log` query parameter in the database connection URL. @@ -220,6 +246,9 @@ Known Limitations in ibm_db_sa adapter for DB2 databases 5) DEFERRABLE INITIALLY deferred not supported 6) Subquery in ON clause of LEFT OUTER JOIN not supported 7) PyODBC support is experimental +8) A unique index created with EXCLUDE NULL KEYS is reflected as a UNIQUE constraint +9) Enum columns are created as VARCHAR(n CODEUNITS32) on Db2 LUW, which requires Db2 LUW 10.5 or later. Db2 for z/OS uses VARCHAR(n) and Db2 for i uses VARCHAR(n) CCSID 1208 +10) UnicodeText columns are created as DBCLOB (previously LONG VARGRAPHIC, which is deprecated in Db2) Credits diff --git a/README.unittests.md b/README.unittests.md new file mode 100644 index 0000000..2445965 --- /dev/null +++ b/README.unittests.md @@ -0,0 +1,263 @@ +## Running the ibm_db_sa Test Suite + +`ibm_db_sa` is tested with SQLAlchemy's own pytest-based test framework, +the same framework SQLAlchemy provides for all third-party dialects (see +SQLAlchemy's [README.unittests.rst](https://github.com/sqlalchemy/sqlalchemy/blob/main/README.unittests.rst)). + +You do not need a copy of the SQLAlchemy source code. The test framework +(`sqlalchemy.testing`) is part of the installed `sqlalchemy` package. + +The test suite runs on SQLAlchemy 1.4.x, 2.0.x and 2.1.x. + +**Contents** + +- [Test files](#test-files) +- [Setup](#setup) +- [Configuring the database connection](#configuring-the-database-connection) +- [Running the tests](#running-the-tests) +- [Testing with a different SQLAlchemy version](#testing-with-a-different-sqlalchemy-version) +- [Investigating a failing test](#investigating-a-failing-test) +- [Common pytest options](#common-pytest-options) + +--- + +### Test files + +| File | Description | Requires Db2 | +|------|-------------|:------------:| +| `conftest.py` | Registers the Db2 dialect and loads SQLAlchemy's pytest plugin. | - | +| `setup.cfg` | Test configuration: requirements class, named database URLs and pytest options. | - | +| `ibm_db_sa/requirements.py` | Lists the features Db2 supports. Unsupported features are skipped. | - | +| `test/test_dialect.py` | SQL compilation and type tests. | No | +| `test/test_suite.py` | SQLAlchemy's dialect compliance suite (`sqlalchemy.testing.suite`). | Yes | +| `test/test_out_parameters.py` | Stored procedure OUT parameters. | Yes | +| `test/test_default_schema.py` | Default schema detection and reflection. | Yes | +| `test/test_numeric.py` | `NUMERIC` / `DECIMAL` handling. | Yes | +| `test/test_float.py` | `FLOAT` / `REAL` / `DOUBLE` handling and reflection. | Yes | +| `test/test_luw_reflection.py` | Db2 LUW reflection and `BINARY` / `XML` / `DECFLOAT` handling. | Yes | + +Some files that require Db2 also contain tests that only compile SQL. Those +tests always run; the database tests in the same file are skipped when no +Db2 connection is configured. + +### Setup + +From the root of the repository, install the dialect in editable mode and +the test dependencies: + +```bash +pip install -e . +pip install -r requirements-test.txt +``` + +`requirements-test.txt` installs `sqlalchemy`, `pytest` and the `ibm_db` +driver. + +### Configuring the database connection + +The database used by the tests can be set in `setup.cfg` or passed on the +command line. A URL passed on the command line takes precedence. + +**Option 1: `setup.cfg`** + +The `[db]` section contains named connection URLs: + +```ini +[db] +default = sqlite:///:memory: +sqlite = sqlite:///:memory: +db2 = ibm_db_sa://user:pass@host:port/DBNAME +``` + +Update the `db2` entry with your Db2 details, then select it with +`--db db2`: + +```bash +pytest --db db2 test/test_suite.py +``` + +`default` is used when neither `--db` nor `--dburi` is given. It is set to +SQLite so that `test/test_dialect.py` can run without a Db2 server. + +> **Note:** A plain `pytest` (without `--db` or `--dburi`) runs against +> SQLite. In that case the compliance suite (`test/test_suite.py`) and the +> Db2-only test classes are reported as `skipped`, not run. + +> **Note:** Do not commit real user names or passwords to `setup.cfg`. + +**Option 2: `--dburi`** + +```bash +pytest --dburi "ibm_db_sa://user:pass@host:port/DBNAME" test/test_suite.py +``` + +**Supported URL formats** + +Both of the following formats select the Db2 dialect: + +```text +ibm_db_sa://user:pass@host:port/DBNAME +db2+ibm_db://user:pass@host:port/DBNAME +``` + +If the run fails at startup with +`ArgumentError: Expected string or URL object, got None`, the URL scheme is +not recognized. Use one of the formats above. + +**Checking which database is used** + +If neither `--db` nor `--dburi` is given, the tests run against SQLite, not +Db2. To confirm that Db2 is being used, check that: + +- the start of the output contains `Adding database URL: ibm_db_sa+...`, or +- test names include the Db2 suffix when run with `-v`, for example + `NumericTest_ibm_db_sa+ibm_db_sa_DB2/...`. + +### Running the tests + +**Without a database** + +`test/test_dialect.py` only compiles SQL statements and never connects to a +database. + +```bash +# all tests in the file +pytest test/test_dialect.py + +# one test class +pytest test/test_dialect.py::CompileTest + +# one test +pytest test/test_dialect.py::CompileTest::test_offset_uses_rownumber + +# tests whose name matches an expression +pytest test/test_dialect.py -k "limit or offset" +``` + +`python run_tests.py` is a shortcut for `pytest` and accepts the same +arguments. Without arguments it runs `test/test_dialect.py`. + +**With a Db2 database** + +Add `--db db2` (or `--dburi `) to the command: + +```bash +# all tests +pytest --db db2 test/ + +# the compliance suite only +pytest --db db2 test/test_suite.py + +# several files +pytest --db db2 test/test_suite.py test/test_out_parameters.py + +# one test class +pytest --db db2 "test/test_suite.py::NumericTest" + +# one test +pytest --db db2 "test/test_suite.py::NumericTest::test_numeric_as_decimal" +``` + +To list the available tests without running them: + +```bash +pytest --db db2 test/test_suite.py --collect-only -q +``` + +**Saving the output to a file** + +```bash +# Linux / macOS +pytest --db db2 test/ 2>&1 | tee test_output.txt +``` + +```powershell +# Windows PowerShell +pytest --db db2 test/ 2>&1 | Tee-Object -FilePath test_output.txt +``` + +### Testing with a different SQLAlchemy version + +Install the SQLAlchemy version you want to test, then run the tests as +usual. No changes to the tests are needed. + +```bash +pip install "sqlalchemy==1.4.*" +pytest --db db2 test/ + +pip install "sqlalchemy==2.0.*" +pytest --db db2 test/ + +pip install "sqlalchemy==2.1.*" +pytest --db db2 test/ +``` + +Check the installed version with: + +```bash +python -c "import sqlalchemy; print(sqlalchemy.__version__)" +``` + +### Investigating a failing test + +A test run ends with a summary such as: + +```text +12 failed, 1450 passed, 630 skipped, 3 errors +``` + +| Result | Meaning | +|--------|---------| +| `passed` | The test succeeded on Db2. | +| `skipped` | The feature is marked as not supported in `ibm_db_sa/requirements.py`, or the test does not apply to Db2. | +| `failed` | The test ran, but the result was not what was expected. | +| `error` | The test could not run, usually because of a problem in setup, a fixture or the database connection. One cause often produces many errors, so look at these first. | + +**1. Run the failing test on its own** + +Copy the class and test name from the output. Use `-k` to avoid typing the +long Db2 suffix in the test name: + +```bash +pytest --db db2 "test/test_suite.py::NumericTest" -k "test_numeric_as_decimal" -x -vv +``` + +- `-x` stops at the first failure. +- `-vv` shows the full assertion and SQL details. +- `-s` shows `print` and driver output, which helps with encoding errors. + +**2. Fix and re-run** + +After changing code under `ibm_db_sa/`, re-run only the tests that failed +last time: + +```bash +pytest --db db2 test/test_suite.py --lf -v +``` + +When they pass, run the complete file once more to make sure nothing else +has broken. + +**3. Features that Db2 does not support** + +If a test fails because Db2 does not support the feature being tested, do +not change the test. Mark the feature as unsupported in +`ibm_db_sa/requirements.py` instead. The test is then reported as +`skipped`, and the results continue to reflect what the dialect actually +supports. + +### Common pytest options + +| Option | Description | +|--------|-------------| +| `-v` | Show the name of each test. | +| `-vv` | Show full assertion and SQL details. | +| `-q` | Show less output. | +| `-x` | Stop at the first failure. | +| `-s` | Show `print` and driver output. | +| `-k "expr"` | Run only tests whose name matches `expr`. | +| `--db NAME` | Use a named URL from the `[db]` section of `setup.cfg`. | +| `--dburi URL` | Use the given database URL. | +| `--lf` | Run only the tests that failed in the previous run. | +| `--ff` | Run the previously failed tests first, then the rest. | +| `--collect-only -q` | List the tests without running them. | diff --git a/conftest.py b/conftest.py new file mode 100644 index 0000000..4b53878 --- /dev/null +++ b/conftest.py @@ -0,0 +1,97 @@ +"""pytest bootstrap for the ibm_db_sa dialect test suite. + +This is the entry point used by SQLAlchemy's testing framework for +third-party dialects (see SQLAlchemy's README.unittests.rst). + +It performs two jobs: + +1. Registers the Db2 dialect entry points so they resolve even when + running directly from a source checkout (before ``pip install``). +2. Loads SQLAlchemy's pytest plugin, which provides the ``--dburi`` + option, the ``config`` object, database fixtures and the machinery + that runs ``sqlalchemy.testing.suite`` against this dialect. + +Run the tests with, for example:: + + pytest --dburi "db2+ibm_db://user:pass@host:50000/DBNAME" + +or select the named URL from ``setup.cfg`` with ``pytest --db db2``. +""" + +from sqlalchemy.dialects import registry + +# Make the dialect importable by name from a source checkout. +# +# NOTE on the extra "doubled" keys below: SQLAlchemy's test provisioning +# (provision.generate_driver_url) rebuilds the URL as +# "+" and then looks up that entry point. +# For a bare "db2://" URL that becomes "db2+db2" -> "db2.db2", and for the +# legacy "ibm_db_sa://" URL it becomes "ibm_db_sa+ibm_db_sa" -> +# "ibm_db_sa.ibm_db_sa". We register those aliases so ALL of these forms +# work as --dburi values: +# db2+ibm_db://... (canonical / recommended) +# db2://... (bare) +# ibm_db_sa://... (legacy) +registry.register("db2", "ibm_db_sa.ibm_db", "DB2Dialect_ibm_db") +registry.register("db2.db2", "ibm_db_sa.ibm_db", "DB2Dialect_ibm_db") +registry.register("db2.ibm_db", "ibm_db_sa.ibm_db", "DB2Dialect_ibm_db") +registry.register("db2.ibm_db_sa", "ibm_db_sa.ibm_db", "DB2Dialect_ibm_db") +registry.register("db2.pyodbc", "ibm_db_sa.pyodbc", "DB2Dialect_pyodbc") +registry.register("db2.pyodbc400", "ibm_db_sa.pyodbc", "AS400Dialect_pyodbc") + +# Backwards-compatible "ibm_db_sa://" style names. +registry.register("ibm_db_sa", "ibm_db_sa.ibm_db", "DB2Dialect_ibm_db") +registry.register("ibm_db_sa.ibm_db", "ibm_db_sa.ibm_db", "DB2Dialect_ibm_db") +registry.register("ibm_db_sa.ibm_db_sa", "ibm_db_sa.ibm_db", "DB2Dialect_ibm_db") +registry.register("ibm_db_sa.pyodbc", "ibm_db_sa.pyodbc", "DB2Dialect_pyodbc") + +# Load the SQLAlchemy pytest plugin. This MUST happen after the registry +# calls above. It wires up --dburi, the `config` fixture, and the suite. +from sqlalchemy.testing.plugin.pytestplugin import * # noqa: E402,F401,F403 + +# Register the Db2 test-provisioning hooks (e.g. temp_table_keyword_args). +# Must happen after the plugin import above so the provisioning registry +# exists. Import for its registration side effects only. +import ibm_db_sa.provision # noqa: E402,F401 + +import os # noqa: E402 + +import pytest # noqa: E402 +from sqlalchemy.testing import config as _sa_config # noqa: E402 +from sqlalchemy.testing.plugin import pytestplugin as _sa_plugin # noqa: E402 + + +# Db2 DECIMAL precision is capped at 31 digits; these upstream cases need +# NUMERIC(38, ...) (SQL0405N / SQL0604N) and have no requirement to gate them. +_DB2_DECIMAL_LIMIT_SKIP_IDS = ( + "test_truediv_divisor[numeric_38_digit_divisor", + "test_truediv_divisor[numeric_tiny_divisor", +) + +# The compliance suite is only meaningful against Db2, not the sqlite default. +_DB2_ONLY_FILES = ("test_suite.py",) + + +def _is_db2_backend(): + db = getattr(_sa_config, "db", None) + return db is not None and db.dialect.name == "ibm_db_sa" + + +def pytest_collection_modifyitems(session, config, items): + # Run SQLAlchemy's hook first: it expands __backend__ classes and applies + # __only_on__ / __requires__ gating. + _sa_plugin.pytest_collection_modifyitems(session, config, items) + + skip_db2_decimal = pytest.mark.skip( + reason="Db2 DECIMAL precision maximum is 31 digits (SQL0405N / SQL0604N)." + ) + skip_not_db2 = pytest.mark.skip( + reason="Requires a Db2 database; run with --db db2 or --dburi ." + ) + is_db2 = _is_db2_backend() + for item in items: + if any(tag in item.nodeid for tag in _DB2_DECIMAL_LIMIT_SKIP_IDS): + item.add_marker(skip_db2_decimal) + if not is_db2 and os.path.basename(str(item.fspath)) in _DB2_ONLY_FILES: + item.add_marker(skip_not_db2) + diff --git a/ibm_db_sa/base.py b/ibm_db_sa/base.py index e677ee1..1948484 100644 --- a/ibm_db_sa/base.py +++ b/ibm_db_sa/base.py @@ -22,6 +22,7 @@ import sys import sqlalchemy import datetime, decimal, re +import itertools from sqlalchemy import types as sa_types from sqlalchemy import schema as sa_schema from sqlalchemy import util @@ -29,6 +30,14 @@ from sqlalchemy.sql.elements import BindParameter from sqlalchemy.sql import compiler from sqlalchemy.sql import operators + +# _de_clone is present in SQLAlchemy 1.4.x-2.1.x; the fallback only covers +# releases that predate it. +try: + from sqlalchemy.sql.compiler import _de_clone +except ImportError: # pragma: no cover + def _de_clone(elements): + return elements from sqlalchemy.engine import default from sqlalchemy import event from sqlalchemy.engine import Engine @@ -318,6 +327,14 @@ class XML(sa_types.Text): class DB2TypeCompiler(compiler.GenericTypeCompiler): + def _server_type(self): + # Per-dialect server name ("AS", "DB2", "DB2/LINUXX8664", ...); None before connect. + return getattr(self.dialect, "dbms_name", None) + + def _is_zos(self): + server = self._server_type() or "" + return server == "DB2" or server.startswith("DSN") + @log_entry_exit def visit_TIMESTAMP(self, type_, **kw): sql = "TIMESTAMP" @@ -350,8 +367,15 @@ def visit_SMALLINT(self, type_, **kw): return sql def visit_BOOLEAN(self, type_, **kw): - sql = "BOOLEAN" - logger.debug(f"Type rendering -> BOOLEAN -> {sql}") + # Db2 for i (AS/400): SQLAlchemy drives booleans as integers + # (supports_native_boolean=False), and a native BOOLEAN column does not + # reliably round-trip those 0/1 values through the CLI driver, so store + # booleans as SMALLINT there. LUW / z-OS keep the native BOOLEAN type. + check_server = self._server_type() + sql = "SMALLINT" if check_server == "AS" else "BOOLEAN" + logger.debug( + f"Type rendering -> BOOLEAN -> serverType={check_server}, sql={sql}" + ) return sql def visit_INT(self, type_, **kw): @@ -478,11 +502,23 @@ def visit_float(self, type_, **kw): logger.debug("Redirecting float to FLOAT") return self.visit_FLOAT(type_, **kw) + def _as400_unicode_ccsid(self, base_sql, graphic=False): + # Db2 for i (AS/400): character columns created with the job's default + # CCSID garble non-ASCII data (it round-trips as U+FFFD). Tag string + # columns with a Unicode CCSID so Unicode/Text/String data round-trips: + # SBCS/MBCS types use UTF-8 (1208); GRAPHIC types use UTF-16 (1200). + # This is a no-op on every other Db2 server (LUW / z-OS unchanged). + if self._server_type() != "AS": + return base_sql + return f"{base_sql} CCSID {1200 if graphic else 1208}" + def visit_unicode(self, type_, **kw): - check_server = getattr(DB2Dialect, "serverType") + check_server = self._server_type() base_sql = self.visit_VARGRAPHIC(type_, **kw) if check_server == "DB2": sql = base_sql + " CCSID 1200" + elif check_server == "AS": + sql = self._as400_unicode_ccsid(base_sql, graphic=True) else: sql = base_sql logger.debug( @@ -492,16 +528,40 @@ def visit_unicode(self, type_, **kw): return sql def visit_unicode_text(self, type_, **kw): - logger.debug("Redirecting unicode_text to LONGVARGRAPHIC") - return self.visit_LONGVARGRAPHIC(type_, **kw) + # Map UnicodeText to DBCLOB instead of the deprecated LONG VARGRAPHIC, + # which round-trips Unicode unreliably across CLI driver versions. + logger.debug("Redirecting unicode_text to DBCLOB") + return self._as400_unicode_ccsid( + self.visit_DBCLOB(type_, **kw), graphic=True + ) + + def visit_enum(self, type_, **kw): + # CODEUNITS32 sizes by characters (Db2 LUW 10.5+); Db2 for i and z/OS reject + # it in column DDL, so emit VARCHAR(n) there (with CCSID 1208 on Db2 for i). + length = type_.length + check_server = self._server_type() + if length: + if check_server == "AS": + sql = f"VARCHAR({length}) CCSID 1208" + elif self._is_zos(): + sql = f"VARCHAR({length})" + else: + sql = f"VARCHAR({length} CODEUNITS32)" + else: + sql = self.visit_VARCHAR(type_, **kw) + logger.debug( + f"Type rendering -> ENUM -> length={length}, " + f"serverType={check_server}, sql={sql}" + ) + return sql def visit_string(self, type_, **kw): logger.debug("Redirecting string to VARCHAR") - return self.visit_VARCHAR(type_, **kw) + return self._as400_unicode_ccsid(self.visit_VARCHAR(type_, **kw)) def visit_TEXT(self, type_, **kw): logger.debug("Redirecting TEXT to CLOB") - return self.visit_CLOB(type_, **kw) + return self._as400_unicode_ccsid(self.visit_CLOB(type_, **kw)) def visit_large_binary(self, type_, **kw): logger.debug("Redirecting large_binary to BLOB") @@ -525,6 +585,46 @@ def get_cte_preamble(self, recursive): logger.debug(f"Generating CTE preamble -> recursive={recursive}") return "WITH" + def _as400_empty_set_null_cast(self, type_): + # Return a typed NULL ("CAST(NULL AS )") that is compatible + # with the compared column, so the always-false / always-true empty-IN + # guard is accepted by Db2 for i. The exact precision is irrelevant + # because the surrounding predicate can never match. + if isinstance(type_, sa_types.Integer): + db_type = "INTEGER" + elif isinstance(type_, (sa_types.Numeric, sa_types.Float)): + db_type = "DECIMAL(31, 8)" + elif isinstance(type_, (sa_types.String, sa_types.Enum)): + db_type = "VARCHAR(1)" + else: + db_type = "VARCHAR(1)" + return f"CAST(NULL AS {db_type})" + + def visit_empty_set_op_expr(self, type_, expand_op, **kw): + # Db2 for i (AS/400) rejects a bare NULL literal inside an IN predicate + # (SQL0206N "NULL is not valid in the context where it is used"). + # Emit a typed NULL via CAST so the empty-set expansion is accepted. + # Every other Db2 server keeps SQLAlchemy's default rendering so the + # LUW / z/OS behavior is unchanged. + if getattr(self.dialect, "dbms_name", None) != "AS": + return super(DB2Compiler, self).visit_empty_set_op_expr( + type_, expand_op, **kw + ) + + nulls = [self._as400_empty_set_null_cast(element) for element in type_] + if expand_op is operators.not_in_op: + if len(type_) > 1: + return "(%s)) OR (1 = 1" % (", ".join(nulls)) + else: + return "%s) OR (1 = 1" % nulls[0] + elif expand_op is operators.in_op: + if len(type_) > 1: + return "(%s)) AND (1 != 1" % (", ".join(nulls)) + else: + return "%s) AND (1 != 1" % nulls[0] + else: + return self.visit_empty_set_expr(type_) + @log_entry_exit def visit_now_func(self, fn, **kw): logger.debug("Rendering NOW function as CURRENT_TIMESTAMP") @@ -532,14 +632,9 @@ def visit_now_func(self, fn, **kw): @log_entry_exit def for_update_clause(self, select, **kw): - # Version-agnostic FOR UPDATE handling (SQLAlchemy 0.7.3 -> 2.0.x). - # - # SQLAlchemy >= 1.0 stores the lock request on ``select._for_update_arg`` - # (a ForUpdateArg object, or None when no locking was requested) and the - # legacy ``select.for_update`` attribute was removed. - # - # SQLAlchemy < 1.0 exposes the legacy ``select.for_update`` attribute - # whose value is False / True / "read" / "nowait" / "read_nowait" / etc. + # Version-agnostic FOR UPDATE handling: SQLAlchemy >= 1.0 uses + # ``select._for_update_arg`` (ForUpdateArg or None); older versions use the + # legacy ``select.for_update`` attribute (False/True/"read"/"read_nowait"/etc.). if hasattr(select, "_for_update_arg"): # Modern API (SQLAlchemy 1.0+ including 2.0.x) for_update_arg = select._for_update_arg @@ -647,13 +742,18 @@ def _render_clause(clause): if offset_text is not None: text += f" OFFSET {offset_text}" logger.debug(f"Applied OFFSET -> {offset_text}") + # Remembered so visit_select can strip exactly this trailing OFFSET. + self._db2_last_offset_text = offset_text logger.debug(f"Generated LIMIT/OFFSET clause -> {text}") return text @log_entry_exit def visit_select(self, select, **kw): try: + self._db2_last_offset_text = None sql_ori = compiler.SQLCompiler.visit_select(self, select, **kw) + # LIMIT/OFFSET is rendered last, so this is this SELECT's own offset. + rendered_offset_text = self._db2_last_offset_text sql_upper = sql_ori.upper() logger.debug("Processing SELECT compilation.") if ("LIMIT" in sql_upper) or ("FETCH FIRST" in sql_upper): @@ -682,13 +782,25 @@ def visit_select(self, select, **kw): sql_ori, flags=re.IGNORECASE, ).strip() - sql_work = re.sub( - r"\s+OFFSET\s+(?:\d+|__\[POSTCOMPILE_[^\]]+\]|:[A-Za-z0-9_]+|\?)\s*$", - "", - sql_work, - flags=re.IGNORECASE, + # Strip only this SELECT's trailing OFFSET (any form, e.g. "1 + 2"), + # so OFFSETs in subqueries/CTEs are left intact. + captured_offset_text = None + offset_suffix = ( + f" OFFSET {rendered_offset_text}" + if rendered_offset_text is not None + else None ) - sql_split = re.split(r"[\s+]FROM ", sql_work, 1) + if offset_suffix and sql_work.endswith(offset_suffix): + captured_offset_text = rendered_offset_text + sql_work = sql_work[: -len(offset_suffix)].rstrip() + else: + _offset_kw = re.search( + r"\s+OFFSET\s+(\S+)\s*$", sql_work, flags=re.IGNORECASE + ) + if _offset_kw is not None: + captured_offset_text = _offset_kw.group(1) + sql_work = sql_work[: _offset_kw.start()].rstrip() + sql_split = re.split(r"[\s+]FROM ", sql_work, maxsplit=1) if len(sql_split) < 2: logger.debug("Unable to split SELECT for OFFSET rewrite.") logger.debug(f"Final SELECT SQL -> {sql_ori}") @@ -738,7 +850,11 @@ def _process_clause_text(clause): ) return self.process(clause, **kw) - offset_text = _process_clause_text(offset_clause_obj) + # Reuse the captured token so the OFFSET bind is not registered twice. + if captured_offset_text: + offset_text = captured_offset_text + else: + offset_text = _process_clause_text(offset_clause_obj) limit_text = _process_clause_text(limit_clause_obj) if offset_text is not None: sql = f'{sql} "{__rownum}" > {offset_text}' @@ -767,13 +883,10 @@ def _process_clause_text(clause): @log_entry_exit def visit_sequence(self, sequence, **kw): try: - schema = sequence.schema - name = sequence.name - logger.debug(f"Rendering sequence -> schema={schema}, name={name}") - if schema: - sql = f"NEXT VALUE FOR {schema}.{name}" - else: - sql = f"NEXT VALUE FOR {name}" + # Use preparer.format_sequence to apply identifier quoting and + # schema_translate_map; manual naming would bypass schema translation. + seq_name = self.preparer.format_sequence(sequence) + sql = f"NEXT VALUE FOR {seq_name}" logger.debug(f"Generated sequence SQL -> {sql}") return sql except Exception as e: @@ -856,7 +969,94 @@ def visit_cast(self, cast, **kw): logger.exception("Stack trace in visit_cast") raise + @log_entry_exit + def visit_truediv_binary(self, binary, operator, **kw): + # True division on Db2: cast the divisor to a bounded-precision DECIMAL + # so the quotient keeps a non-zero scale (otherwise a bare Numeric() + # becomes DECIMAL(31, 0) and "15 / 10" yields 1 instead of 1.5). An + # existing Numeric/Float divisor type is reused to preserve its scale. + if getattr(self.dialect, "div_is_floordiv", False): + from sqlalchemy.sql import elements + right = binary.right + affinity = right.type._type_affinity + if affinity in (sa_types.Numeric, sa_types.Float): + cast_type = right.type + else: + # Bounded precision (< 31) leaves Db2 room to compute a + # fractional quotient scale, and a non-zero scale preserves a + # fractional divisor whose type affinity was lost - e.g. a + # divisor wrapped in nullif(...) in the "untyped_function" + # case, where DECIMAL(19, 0) would truncate 3.5 to 3. + cast_type = sa_types.Numeric(19, 6) + sql = ( + self.process(binary.left, **kw) + + " / " + + self.process(elements.Cast(right, cast_type), **kw) + ) + logger.debug(f"Rendered true-division SQL -> {sql}") + return sql + # SQLAlchemy 1.4 has no base visit_truediv_binary; render a plain "/". + if not hasattr(compiler.SQLCompiler, "visit_truediv_binary"): + return self._generate_generic_binary( + binary, compiler.OPERATORS[operator], **kw + ) + return super(DB2Compiler, self).visit_truediv_binary( + binary, operator, **kw + ) + + @log_entry_exit + def visit_floordiv_binary(self, binary, operator, **kw): + # Floor division on Db2: cast both integer operands to INTEGER so Db2 + # does integer division even for bound params. Non-integer operands + # fall through to the base FLOOR(...) implementation. + left_type = binary.left.type._type_affinity + right_type = binary.right.type._type_affinity + if ( + getattr(self.dialect, "div_is_floordiv", False) + and left_type is sa_types.Integer + and right_type is sa_types.Integer + ): + from sqlalchemy.sql import elements + left = self.process( + elements.Cast(binary.left, sa_types.Integer()), **kw + ) + right = self.process( + elements.Cast(binary.right, sa_types.Integer()), **kw + ) + sql = left + " / " + right + logger.debug(f"Rendered integer floor-division SQL -> {sql}") + return sql + return super(DB2Compiler, self).visit_floordiv_binary( + binary, operator, **kw + ) + + @log_entry_exit + def render_literal_value(self, value, type_): + # ibm_db can't carry non-ASCII chars rendered inline, so emit them as a + # Db2 ASCII-only Unicode constant UX''. ASCII strings and + # non-string values defer to the base impl for normal quoting/escaping. + if isinstance(value, str) and not value.isascii(): + hex_utf16 = value.encode("utf-16-be").hex().upper() + sql = "UX'%s'" % hex_utf16 + logger.debug("Rendering non-ASCII literal as Db2 UX constant") + return sql + return super(DB2Compiler, self).render_literal_value(value, type_) + def get_select_precolumns(self, select, **kwargs): + # DISTINCT ON is PostgreSQL-only. When a non-empty _distinct_on is + # present, mirror SQLAlchemy core's behaviour and emit the + # deprecation warning (the ON expression is otherwise silently + # ignored and a plain DISTINCT is rendered). Without this, the ON + # expression would be dropped with no feedback. + if getattr(select, "_distinct_on", None): + util.warn_deprecated( + "DISTINCT ON is currently supported only by the PostgreSQL " + "dialect. Use of DISTINCT ON for other backends is currently " + "silently ignored, however this usage is deprecated, and will " + "raise CompileError in a future release for all backends " + "that do not support this syntax.", + version="1.4", + ) distinct_value = select._distinct if isinstance(distinct_value, str): result = distinct_value.upper() + " " @@ -868,8 +1068,18 @@ def get_select_precolumns(self, select, **kwargs): return result @log_entry_exit - def visit_join(self, join, asfrom=False, **kwargs): + def visit_join(self, join, asfrom=False, from_linter=None, **kwargs): try: + # Register the join's left/right edge with the FROM linter to + # avoid a false cartesian-product warning when the ON clause + # references no columns (e.g. true()/false()). + if from_linter: + from_linter.edges.update( + itertools.product( + _de_clone(join.left._from_objects), + _de_clone(join.right._from_objects), + ) + ) join_type = " INNER JOIN " if join.full: join_type = " FULL OUTER JOIN " @@ -881,11 +1091,19 @@ def visit_join(self, join, asfrom=False, **kwargs): ) sql = "".join( ( - self.process(join.left, asfrom=True, **kwargs), + self.process( + join.left, asfrom=True, + from_linter=from_linter, **kwargs + ), join_type, - self.process(join.right, asfrom=True, **kwargs), + self.process( + join.right, asfrom=True, + from_linter=from_linter, **kwargs + ), " ON ", - self.process(join.onclause, **kwargs), + self.process( + join.onclause, from_linter=from_linter, **kwargs + ), ) ) logger.debug(f"Generated JOIN SQL -> {sql}") @@ -895,6 +1113,73 @@ def visit_join(self, join, asfrom=False, **kwargs): logger.exception("Stack trace in visit_join") raise + @log_entry_exit + def visit_bindparam(self, bindparam, within_columns_clause=False, **kw): + text = super(DB2Compiler, self).visit_bindparam( + bindparam, within_columns_clause=within_columns_clause, **kw + ) + # Wrap typed binds in a CAST so Db2 can infer the marker's type: + # untyped markers in the SELECT projection raise SQL0418N, and + # BigInteger binds default to 32-bit INTEGER (overflowing for large + # ints), so CAST them to BIGINT regardless of clause position. + try: + is_bigint = isinstance(bindparam.type, sa_types.BigInteger) + if ( + (within_columns_clause or is_bigint) + and not kw.get("literal_binds", False) + and "[POSTCOMPILE" not in text + ): + type_ = bindparam.type + # Skip unknown types and variable-length string/binary types + # that have no length, which cannot be a bare CAST target in + # Db2 (e.g. "CAST(? AS VARCHAR)" is invalid without a length). + length_missing = ( + isinstance(type_, (sa_types.String, sa_types.LargeBinary)) + and getattr(type_, "length", None) is None + ) + # Skip CAST for Numeric binds without precision: Db2 renders + # them as DECIMAL(31, 0), truncating fractional values (Float + # excluded as it maps to a binary floating type). + numeric_without_precision = ( + isinstance(type_, sa_types.Numeric) + and not isinstance(type_, sa_types.Float) + and getattr(type_, "precision", None) is None + ) + if ( + type_ is not None + and not isinstance(type_, sa_types.NullType) + and not length_missing + and not numeric_without_precision + ): + type_compiler = getattr( + self.dialect, "type_compiler_instance", None + ) or self.dialect.type_compiler + type_text = type_compiler.process(type_) + casted = f"CAST({text} AS {type_text})" + logger.debug(f"Wrapped column-clause bind -> {casted}") + return casted + except (exc.CompileError, NotImplementedError): + logger.exception( + "Failed to add Db2 bind CAST; falling back to plain marker" + ) + return text + + @log_entry_exit + def visit_frame_clause(self, frameclause, **kw): + # Db2 requires literal integers in window frame bounds (parameter + # markers raise SQL0104N); force literal_execute so binds render inline + # without baking values into the cache key. SQLAlchemy 2.1+ only. + kw["literal_execute"] = True + return super(DB2Compiler, self).visit_frame_clause(frameclause, **kw) + + @log_entry_exit + def _format_frame_clause(self, range_, **kw): + # SQLAlchemy 1.4 / 2.0 window frame bounds: Db2 rejects parameter markers + # here (SQL0104N), so force literal_execute to render bounds inline + # (e.g. "ROWS BETWEEN 5 PRECEDING ...") while keeping them out of the cache key. + kw["literal_execute"] = True + return super(DB2Compiler, self)._format_frame_clause(range_, **kw) + @log_entry_exit def visit_savepoint(self, savepoint_stmt): sid = self.preparer.format_savepoint(savepoint_stmt) @@ -941,6 +1226,33 @@ def visit_unary(self, unary, **kw): class DB2DDLCompiler(compiler.DDLCompiler): + @log_entry_exit + def visit_create_table_as(self, element, **kw): + # Db2 CTAS form: "CREATE TABLE t AS (SELECT ...) WITH DATA" (SQLAlchemy 2.1+). + if getattr(element, "temporary", False): + raise exc.CompileError( + "Db2 dialect does not support CREATE TEMPORARY TABLE ... AS" + ) + if getattr(element, "if_not_exists", False): + raise exc.CompileError( + "Db2 dialect does not support CREATE TABLE IF NOT EXISTS ... AS" + ) + prep = self.preparer + inner_kw = dict(kw) + # Inline literals so the DDL is a self-contained statement. + inner_kw["literal_binds"] = True + select_sql = self.sql_compiler.process(element.selectable, **inner_kw) + parts = [ + "CREATE TABLE", + prep.format_table(element.table), + "AS", + f"({select_sql})", + "WITH DATA", + ] + sql = " ".join(parts) + logger.debug(f"Generated CREATE TABLE AS SQL -> {sql}") + return sql + @staticmethod @log_entry_exit def get_server_version_info(dialect): @@ -1240,7 +1552,10 @@ def visit_add_constraint(self, create, **kw): class DB2IdentifierPreparer(compiler.IdentifierPreparer): reserved_words = RESERVED_WORDS - illegal_initial_characters = set(range(0, 10)).union(["_", "$"]) + # Db2 regular identifiers must begin with a letter, so a leading digit, + # underscore or "$" forces quoting. Must be single-character strings (not + # ints) since _requires_quotes tests ``value[0] in illegal_initial_characters``. + illegal_initial_characters = set("0123456789").union(["_", "$"]) def __init__(self, dialect): logger.debug("Initializing DB2IdentifierPreparer") super(DB2IdentifierPreparer, self).__init__(dialect) @@ -1251,21 +1566,15 @@ def __init__(self, dialect): ) def quote_identifier(self, value): - """Override to uppercase normalized identifiers before quoting. - Db2 folds unquoted identifiers to uppercase, so when we need to quote - a name that was stored as lowercase (SQLAlchemy's normalized form), - we must uppercase it to match the catalog (e.g., "ROLE" not "role"). - - Only case-insensitive names are uppercased. If the name is a SQLAlchemy - ``quoted_name`` that was explicitly marked to preserve quoting - (``quote=True``) — i.e. a deliberately case-sensitive identifier — it is - left exactly as-is so lowercase-quoted objects still resolve correctly. - Mixed-case names are likewise preserved. - """ + """Quote an identifier, folding plain lowercase names to Db2's UPPERCASE form.""" force_quote = getattr(value, "quote", None) if force_quote is not True and value == value.lower(): value = value.upper() - return self.initial_quote + value + self.final_quote + return ( + self.initial_quote + + self._escape_identifier(value) + + self.final_quote + ) class _SelectLastRowIDMixin(object): @@ -1349,8 +1658,20 @@ class DB2ExecutionContext(_SelectLastRowIDMixin, default.DefaultExecutionContext def fire_sequence(self, seq, type_): sequence_name = str(seq) try: - formatted_seq = self.dialect.identifier_preparer.format_sequence(seq) + # Use the schema_translate_map-aware execution-context preparer, + # not the dialect's default preparer, to avoid SQL0204N when a + # schema_translate_map is active. + preparer = self.identifier_preparer + formatted_seq = preparer.format_sequence(seq) sql = ("SELECT NEXTVAL FOR " + formatted_seq + " FROM SYSIBM.SYSDUMMY1") + # Substitute the __[SCHEMA_x] placeholder with the real schema for this raw statement. + schema_translate_map = self.execution_options.get( + "schema_translate_map", None + ) + if schema_translate_map: + sql = preparer._render_schema_translates( + sql, schema_translate_map + ) logger.debug(f"Firing sequence -> name={sequence_name}, Generated SQL={sql}") result = self._execute_scalar(sql, type_) logger.info(f"Sequence value generated -> name={sequence_name}, value={result}") @@ -1383,6 +1704,9 @@ class DB2Dialect(default.DefaultDialect): supports_native_decimal = False supports_native_boolean = False supports_statement_cache = True + # Db2 persists table/column comments via separate COMMENT ON statements + # (no inline syntax), stored in SYSCAT.*.REMARKS. + supports_comments = True preexecute_sequences = False supports_alter = True supports_sequences = True @@ -1426,6 +1750,7 @@ def initialize(self, connection): logger.info( f"Connected to DB Server -> name={self.dbms_name}, version={self.dbms_ver}" ) + # Kept for backward compatibility only; compilers read self.dialect.dbms_name. DB2Dialect.serverType = self.dbms_name super(DB2Dialect, self).initialize(connection) logger.debug( diff --git a/ibm_db_sa/ibm_db.py b/ibm_db_sa/ibm_db.py index 6d017be..f267c66 100644 --- a/ibm_db_sa/ibm_db.py +++ b/ibm_db_sa/ibm_db.py @@ -19,6 +19,7 @@ import re import warnings +import decimal from collections.abc import MutableMapping, Sequence from typing import Any, cast @@ -46,6 +47,7 @@ SQL_TXN_SERIALIZABLE = 8 SQL_ATTR_TXN_ISOLATION = 108 + class _IBM_Numeric_ibm_db(sa_types.Numeric): @log_entry_exit def result_processor(self, dialect, coltype): @@ -64,6 +66,17 @@ def to_float(value): return to_float +class _IBM_Float_ibm_db(sa_types.Float): + """Float mapped explicitly so it keeps Float's own result handling rather + than being adapted to the Numeric colspec. + + The bind side is left as pass-through (SQLAlchemy's native behaviour): the + executemany homogeneity coercion that ibm_db needs is applied centrally in + DB2Dialect_ibm_db.do_executemany, not per value, so a single bound value is + never altered (its exact Python type/precision is preserved). + """ + + class _IBM_Binary_ibm_db(sa_types._Binary): """Bind binary values as bytes. @@ -157,6 +170,9 @@ def get_result_proxy(self): class DB2Dialect_ibm_db(DB2Dialect): driver = 'ibm_db_sa' supports_unicode_statements = True + # Pass str binds straight to ibm_db, which handles VARGRAPHIC/DBCLOB natively; + # with False, SA 1.4 pre-encodes str binds to UTF-8 bytes. + supports_unicode_binds = True supports_statement_cache = True supports_sane_rowcount = True supports_sane_multi_rowcount = False @@ -176,10 +192,9 @@ class DB2Dialect_ibm_db(DB2Dialect): # DECFLOAT is a Numeric but keeps its own processors. DECFLOAT: DECFLOAT, XML: _IBM_XML_ibm_db, - # Float subclasses Numeric; without its own entry it would be - # adapted to _IBM_Numeric_ibm_db, which ignores Float's - # asdecimal result conversion. - sa_types.Float: sa_types.Float, + # Keep Float's own result handling (and executemany coercion) + # instead of being adapted to _IBM_Numeric_ibm_db. + sa_types.Float: _IBM_Float_ibm_db, } ) @@ -188,19 +203,31 @@ class DB2Dialect_ibm_db(DB2Dialect): @log_entry_exit def dbapi(cls): """ Returns: the underlying DBAPI driver module + + SQLAlchemy < 2.0 core calls the ``dbapi()`` classmethod to load the + DBAPI module, so it is kept here. """ logger.debug("Importing ibm_db_dbi DBAPI module") import ibm_db_dbi as module return module - else: + @classmethod @log_entry_exit def import_dbapi(cls): - """ Returns: the underlying DBAPI driver module + """ Returns the underlying DBAPI driver module. Provided under the + 2.0 name ``import_dbapi()`` so callers resolve it on < 2.0 too. """ logger.debug("Importing ibm_db_dbi DBAPI module") import ibm_db_dbi as module return module + else: + @classmethod + @log_entry_exit + def import_dbapi(cls): + """ Returns the underlying DBAPI driver module """ + logger.debug("Importing ibm_db_dbi DBAPI module") + import ibm_db_dbi as module + return module @log_entry_exit def do_execute(self, cursor, statement, parameters, context=None): @@ -212,8 +239,7 @@ def do_execute(self, cursor, statement, parameters, context=None): statement = statement.split('(', 1)[0].split()[1] context._callproc_result = cursor.callproc(statement, parameters) else: - check_server = getattr(DB2Dialect, 'serverType') - if ("round(" in statement.casefold()) and check_server == "DB2": + if ("round(" in statement.casefold()) and self.dbms_name == "DB2": logger.debug("Applying round() workaround for DB2") value_index = 0 while '?' in statement and value_index < len(parameters): @@ -223,6 +249,96 @@ def do_execute(self, cursor, statement, parameters, context=None): else: cursor.execute(statement, parameters) + @staticmethod + def _bind_types(context): + """Return the positional bind types of a compiled statement, or None.""" + compiled = getattr(context, "compiled", None) + positiontup = getattr(compiled, "positiontup", None) + bind_names = getattr(compiled, "bind_names", None) + if not positiontup or not bind_names: + return None + by_name = {name: bind for bind, name in bind_names.items()} + if any(name not in by_name for name in positiontup): + return None + return [by_name[name].type for name in positiontup] + + @staticmethod + def _homogenize_numeric_columns(parameters, bind_types=None): + """Coerce executemany columns that mix int/float/Decimal to one Python + type so ibm_db's bulk bind accepts the batch; uniform columns are left as-is. + """ + if not isinstance(parameters, (list, tuple)) or len(parameters) < 2: + return parameters + if not all(isinstance(row, (list, tuple)) for row in parameters): + return parameters + ncols = len(parameters[0]) + if ncols == 0 or any(len(row) != ncols for row in parameters): + return parameters + if bind_types is not None and len(bind_types) != ncols: + bind_types = None + + rows = None + for col in range(ncols): + col_type = bind_types[col] if bind_types is not None else None + # With known bind types, only Numeric/Float columns are considered + # (Float is not a Numeric subclass on SQLAlchemy 2.1). + if col_type is not None and not isinstance(col_type, sa_types.NullType): + if not isinstance(col_type, (sa_types.Numeric, sa_types.Float)): + continue + saw_decimal = saw_float = saw_int = saw_other = False + for row in parameters: + value = row[col] + if value is None: + continue + if isinstance(value, bool): + saw_other = True + elif isinstance(value, decimal.Decimal): + saw_decimal = True + elif isinstance(value, float): + saw_float = True + elif isinstance(value, int): + saw_int = True + else: + saw_other = True + if saw_other: + continue + if sum((saw_decimal, saw_float, saw_int)) < 2: + continue # already a single numeric type (or all None) + # DECIMAL columns use Decimal (lossless); FLOAT columns use float, as ibm_db + # on SA 1.4 can't bind Decimal to FLOAT/DOUBLE (CLI0100E). + if isinstance(col_type, sa_types.Numeric) and not isinstance( + col_type, sa_types.Float + ): + target = decimal.Decimal + elif saw_decimal and SA_VERSION_MM >= (2, 0): + target = decimal.Decimal + else: + target = float + if rows is None: + rows = [list(row) for row in parameters] + for row in rows: + value = row[col] + if value is None or isinstance(value, target): + continue + row[col] = ( + decimal.Decimal(str(value)) + if target is decimal.Decimal + else float(value) + ) + + if rows is None: + return parameters + return [tuple(row) for row in rows] + + @log_entry_exit + def do_executemany(self, cursor, statement, parameters, context=None): + logger.debug("Executing SQL statement (executemany)") + parameters = self._homogenize_numeric_columns( + parameters, self._bind_types(context) + ) + cursor.executemany(statement, parameters) + + @log_entry_exit def _get_server_version_info(self, connection): logger.debug("Fetching DB2 server version") @@ -348,9 +464,7 @@ def create_connect_args( @log_entry_exit def _get_default_schema_name(self, connection): logger.debug("Fetching current schema from DB2") - # Ask the server. ibm_db_dbi's get_current_schema() falls back to the - # user argument passed to connect(), which is empty when credentials - # are supplied in the DSN, so it can return '' for a valid session. + # Ask the server directly, since ibm_db_dbi's get_current_schema() can return '' for DSN-based credentials. if hasattr(connection, "exec_driver_sql"): result = connection.exec_driver_sql("VALUES CURRENT SCHEMA") else: # SQLAlchemy < 1.4 diff --git a/ibm_db_sa/provision.py b/ibm_db_sa/provision.py new file mode 100644 index 0000000..5761aad --- /dev/null +++ b/ibm_db_sa/provision.py @@ -0,0 +1,34 @@ +# +--------------------------------------------------------------------------+ +# | Licensed Materials - Property of IBM | +# | | +# | (C) Copyright IBM Corporation 2008, 2016. | +# +--------------------------------------------------------------------------+ +# | Licensed under the Apache License, Version 2.0 (the "License"); | +# | you may not use this file except in compliance with the License. | +# +--------------------------------------------------------------------------+ +"""Test-only provisioning hooks for the ibm_db_sa (Db2) dialect. + +Imported only by SQLAlchemy's testing framework, so it has no effect at +runtime. It registers ``temp_table_keyword_args``, which has no default +implementation and otherwise raises ``NotImplementedError`` during +temp-table reflection fixtures. The import is guarded to stay a no-op on +SQLAlchemy versions where the symbol is missing. +""" + +try: + from sqlalchemy.testing.provision import temp_table_keyword_args +except Exception: # pragma: no cover - SQLAlchemy without the provisioning hook + temp_table_keyword_args = None + + +if temp_table_keyword_args is not None: + + # Register under both "ibm_db_sa" (dialect name) and "db2" (conftest alias) + # so provisioning resolves regardless of the active backend name. + @temp_table_keyword_args.for_db("ibm_db_sa", "db2") + def _db2_temp_table_keyword_args(cfg, eng): + """Return the ``GLOBAL TEMPORARY`` prefix so SQLAlchemy emits valid + Db2 ``CREATE GLOBAL TEMPORARY TABLE`` DDL for temp-table fixtures. + """ + return {"prefixes": ["GLOBAL TEMPORARY"]} + diff --git a/ibm_db_sa/reflection.py b/ibm_db_sa/reflection.py index e3fca87..0aee70b 100644 --- a/ibm_db_sa/reflection.py +++ b/ibm_db_sa/reflection.py @@ -21,6 +21,8 @@ from sqlalchemy import sql, util, join from sqlalchemy import Table, MetaData, Column from sqlalchemy.engine import reflection +from sqlalchemy.exc import NoSuchTableError +from sqlalchemy.sql.elements import quoted_name from sqlalchemy import * from .logger import logger, log_entry_exit import re @@ -53,20 +55,17 @@ def __init__(self, dialect): def normalize_name(self, name): try: original_name = name - if isinstance(name, str): - name = name if name is not None: - requires_quotes = self.identifier_preparer._requires_quotes( - name.lower() - ) - result = ( - name.lower() - if name.upper() == name and not requires_quotes - else name - ) + if name.upper() == name: + # Db2 stores unquoted identifiers in UPPERCASE; fold them to + # lowercase to match SQLAlchemy models (incl. words like REF). + result = name.lower() + else: + # Non-uppercase names come from force-quoted identifiers; + # preserve case and mark quoted so they stay case-sensitive. + result = quoted_name(name, quote=True) logger.debug( f"normalize_name -> original={original_name}, " - f"requires_quotes={requires_quotes}, " f"result={result}" ) return result @@ -84,12 +83,12 @@ def denormalize_name(self, name): if name is None: logger.debug("denormalize_name -> input is None") return None - lower_name = name.lower() - requires_quotes = self.identifier_preparer._requires_quotes( - lower_name - ) - if lower_name == name and not requires_quotes: - name = name.upper() + # Force-quoted identifiers (quote=True) are stored case-sensitively + # exactly as written, so leave their case alone. Only ordinary + # lowercase names are folded to Db2's stored UPPERCASE form. + if not (isinstance(name, quoted_name) and name.quote): + if name.lower() == name: + name = name.upper() supports_unicode = self.dialect.supports_unicode_binds if not supports_unicode: if isinstance(name, str): @@ -103,7 +102,6 @@ def denormalize_name(self, name): name = str(name) logger.debug( f"denormalize_name -> original={original_name}, " - f"requires_quotes={requires_quotes}, " f"supports_unicode_binds={supports_unicode}, " f"result={name}" ) @@ -154,6 +152,15 @@ def default_schema_name(self): ) return schema_name + def _ensure_table_exists(self, connection, table_name, schema, **kw): + """Raise NoSuchTableError if the table/view is absent. + + Pass the ORIGINAL (not denormalized) table_name, since has_table + denormalizes internally; views count as existing. + """ + if not self.has_table(connection, table_name, schema=schema, **kw): + raise NoSuchTableError(table_name) + class DB2Reflector(BaseReflector): ischema = MetaData() @@ -186,6 +193,7 @@ class DB2Reflector(BaseReflector): Column("SYSTEM_REQUIRED", sa_types.SMALLINT, key="system_required"), Column("INDSCHEMA", CoerceUnicode, key="indschema"), Column("INDEXTYPE", CoerceUnicode, key="indextype"), + Column("NULLKEYS", CoerceUnicode, key="nullkeys"), schema="SYSCAT") sys_indexcoluse = Table("INDEXCOLUSE", ischema, @@ -271,6 +279,7 @@ class DB2Reflector(BaseReflector): Column("SEQNAME", CoerceUnicode, key="seqname"), schema="SYSCAT") + @reflection.cache @log_entry_exit def has_table(self, connection, table_name, schema=None, **kw): try: @@ -306,9 +315,13 @@ def has_table(self, connection, table_name, schema=None, **kw): logger.exception("Stack trace in has_table") raise + @reflection.cache @log_entry_exit def has_sequence(self, connection, sequence_name, schema=None, **kw): # SQLAlchemy 2's Inspector passes info_cache and other keywords. + # @reflection.cache makes the result participate in the Inspector's + # cache (keyed via info_cache) so repeated has_sequence() calls return + # the cached value until Inspector.clear_cache() is invoked. current_schema = self.denormalize_name(schema or self.default_schema_name) sequence_name = self.denormalize_name(sequence_name) if current_schema: @@ -433,6 +446,7 @@ def get_view_names(self, connection, schema=None, **kw): @log_entry_exit def get_view_definition(self, connection, viewname, schema=None, **kw): try: + original_viewname = viewname current_schema = self.denormalize_name(schema or self.default_schema_name) viewname = self.denormalize_name(viewname) logger.debug(f"Fetching view definition -> schema={current_schema}, view={viewname}") @@ -443,6 +457,9 @@ def get_view_definition(self, connection, viewname, schema=None, **kw): ) logger.debug(f"Generated get_view_definition SQL -> {query}") definition = connection.execute(query).scalar() + if definition is None: + # No SYSCAT.VIEWS row: not a view, so raise per SQLAlchemy's contract. + raise NoSuchTableError(original_viewname) logger.debug(f"View definition length -> {len(definition) if definition else 0}") return definition except Exception as e: @@ -453,6 +470,7 @@ def get_view_definition(self, connection, viewname, schema=None, **kw): @reflection.cache @log_entry_exit def get_columns(self, connection, table_name, schema=None, **kw): + self._ensure_table_exists(connection, table_name, schema, **kw) current_schema = self.denormalize_name(schema or self.default_schema_name) table_name = self.denormalize_name(table_name) syscols = self.sys_columns @@ -533,6 +551,7 @@ def _constraint_columns(self, connection, table_name, schema, kind): def get_pk_constraint(self, connection, table_name, schema=None, **kw): # Read the constraint's own name and columns in key order; splitting # SYSCAT.INDEXES.COLNAMES on \w+ broke names such as "AMT$X". + self._ensure_table_exists(connection, table_name, schema, **kw) rows = self._constraint_columns(connection, table_name, schema, 'P') return { "constrained_columns": [col for _, col in rows], @@ -576,6 +595,7 @@ def get_foreign_keys(self, connection, table_name, schema=None, **kw): # Scope the constrained table by schema and name, and pair columns by # key position, so a same-named table in another schema does not # contribute its keys and composite keys keep their order. + self._ensure_table_exists(connection, table_name, schema, **kw) default_schema = self.normalize_name(self.default_schema_name) current_schema = self.denormalize_name(schema or self.default_schema_name) table_name = self.denormalize_name(table_name) @@ -696,11 +716,12 @@ def get_indexes(self, connection, table_name, schema=None, **kw): # Read columns from SYSCAT.INDEXCOLUSE (splitting COLNAMES on \w+ # broke names such as "AMT$X" and lost DESC), and skip DB2's internal # XML region/path indexes, which are not user indexes. + self._ensure_table_exists(connection, table_name, schema, **kw) current_schema = self.denormalize_name(schema or self.default_schema_name) table_name = self.denormalize_name(table_name) idx, cols = self.sys_indexes, self.sys_indexcoluse query = ( - sql.select(idx.c.indname, idx.c.uniquerule, cols.c.colname, cols.c.colorder) + sql.select(idx.c.indname, idx.c.uniquerule, idx.c.nullkeys, cols.c.colname, cols.c.colorder) .select_from(join(idx, cols, and_( cols.c.indschema == idx.c.indschema, cols.c.indname == idx.c.indname, @@ -717,12 +738,17 @@ def get_indexes(self, connection, table_name, schema=None, **kw): .order_by(idx.c.indname, cols.c.colseq) ) indexes = {} - for name, rule, col, order in connection.execute(query): + for name, rule, nullkeys, col, order in connection.execute(query): index = indexes.setdefault(name, { 'name': self.normalize_name(name), 'column_names': [], 'unique': rule == 'U', }) + # A NULLKEYS='N' unique index backs a UNIQUE constraint (already + # reported by get_unique_constraints), so mark it as a duplicate to + # avoid a separate Index and a name clash. + if rule == 'U' and nullkeys == 'N': + index['duplicates_constraint'] = self.normalize_name(name) index['column_names'].append(self.normalize_name(col)) if order == 'D': index.setdefault('column_sorting', {})[self.normalize_name(col)] = ('desc',) @@ -732,10 +758,45 @@ def get_indexes(self, connection, table_name, schema=None, **kw): @log_entry_exit def get_unique_constraints(self, connection, table_name, schema=None, **kw): constraints = {} + self._ensure_table_exists(connection, table_name, schema, **kw) for name, col in self._constraint_columns(connection, table_name, schema, 'U'): constraints.setdefault(name, []).append(col) + # UNIQUE constraints with a nullable column are stored by Db2 as unique + # indexes (NULLKEYS='N'); report them here so they reflect as constraints. + for name, cols in self._unique_null_key_indexes(connection, table_name, schema): + constraints.setdefault(name, cols) return [{'name': k, 'column_names': v} for k, v in constraints.items()] + def _unique_null_key_indexes(self, connection, table_name, schema): + """Return (index name, [columns]) for unique indexes with NULLKEYS='N', + which back UNIQUE constraints on nullable columns. Plain user indexes + (NULLKEYS='Y') are excluded. + """ + current_schema = self.denormalize_name(schema or self.default_schema_name) + table_name = self.denormalize_name(table_name) + idx, cols = self.sys_indexes, self.sys_indexcoluse + query = ( + sql.select(idx.c.indname, cols.c.colname) + .select_from(join(idx, cols, and_( + cols.c.indschema == idx.c.indschema, + cols.c.indname == idx.c.indname, + ))) + .where(and_( + idx.c.tabschema == current_schema, + idx.c.tabname == table_name, + idx.c.uniquerule == 'U', + idx.c.system_required == 0, + idx.c.nullkeys == 'N', + )) + .order_by(idx.c.indname, cols.c.colseq) + ) + result = {} + for name, col in connection.execute(query): + result.setdefault( + self.normalize_name(name), [] + ).append(self.normalize_name(col)) + return list(result.items()) + @reflection.cache @log_entry_exit def get_check_constraints(self, connection, table_name, schema=None, **kw): diff --git a/ibm_db_sa/requirements.py b/ibm_db_sa/requirements.py index 4b2043a..cb127f8 100644 --- a/ibm_db_sa/requirements.py +++ b/ibm_db_sa/requirements.py @@ -1,8 +1,10 @@ """requirements.py -This file is used by the SQLAlchemy 0.8 testing suite to mark various -optional behaviors as non-supported. +This file is used by the SQLAlchemy test suite (``sqlalchemy.testing``) to +mark various optional behaviors as supported or non-supported for the Db2 +dialect. Open an exclusion to opt in to a capability, close it to skip the +corresponding compliance tests. """ from sqlalchemy.testing.requirements import SuiteRequirements @@ -11,6 +13,53 @@ class Requirements(SuiteRequirements): + @property + def views(self): + """Target database must support VIEWs. + + Db2 fully supports CREATE VIEW, and the dialect's reflector implements + ``get_view_names`` and ``get_view_definition`` (querying SYSCAT.VIEWS). + The base ``SuiteRequirements.views`` is closed by default, which meant + the ComponentReflectionTest view fixtures (``define_views``) never ran, + so the ungated ``test_get_view_names`` expected three views but found + none. Opening this enables view creation and the view-reflection tests + (``view_reflection`` and ``view_column_reflection`` both derive from + this property). + """ + + return exclusions.open() + + @property + def reflects_pk_names(self): + """Target driver reflects the name of primary key constraints. + + Db2 stores primary-key constraint names in the catalog (SYSCAT.TABCONST) + and the dialect's ``get_pk_constraint`` returns that name. The base + ``SuiteRequirements.reflects_pk_names`` is closed by default, which made + ``test_get_pk_constraint`` wrap its name assertion in ``fail_if()`` and + then report "Unexpected success" because the dialect does reflect the + name correctly. Opening this both fixes that test and un-skips + ``test_get_pk_constraint_quoted_name`` (PK names with mixed case / dots, + handled by the dialect's identifier normalization). + """ + + return exclusions.open() + + @property + def unique_constraints_reflect_as_index(self): + """Target database reflects unique constraints as indexes. + + Db2 does not allow a nullable column in a true UNIQUE *constraint*, so + for Db2 >= 10.5 the dialect creates a UNIQUE constraint that includes a + nullable column as a unique INDEX with EXCLUDE NULL KEYS. get_indexes + reports that index (flagged with ``duplicates_constraint``) and + get_unique_constraints reports the matching constraint, so such unique + constraints do round-trip as indexes. Foreign keys, by contrast, are not + auto-indexed by Db2, so ``foreign_keys_reflect_as_index`` stays closed. + """ + + return exclusions.open() + @property def on_update_cascade(self): """"target database must support ON UPDATE..CASCADE behavior in @@ -18,6 +67,33 @@ def on_update_cascade(self): return exclusions.closed() + @property + def implicit_decimal_binds(self): + """Selecting a bare Decimal bind, e.g. ``select(literal(Decimal(...)))``, + returns it as a Decimal. + + Db2 rejects an untyped parameter marker in this position with SQL0418N + ("invalid use of an untyped parameter marker"): ``SELECT ? FROM + SYSIBM.SYSDUMMY1`` has no context from which Db2 can infer the type. + The equivalent with an explicit cast works (and + test_decimal_coerce_round_trip_w_cast passes), so this is specifically + the uncast form Db2 does not support. + """ + + return exclusions.closed() + + @property + def literal_float_coercion(self): + """Selecting a bare float bind, e.g. ``select(literal(15.7563))``, + returns the exact value. + + Same root cause as implicit_decimal_binds: Db2 raises SQL0418N for the + untyped parameter marker in ``SELECT ? FROM SYSIBM.SYSDUMMY1``. The cast + form works, so only the uncast float coercion is unsupported. + """ + + return exclusions.closed() + @property def datetime_microseconds(self): """target dialect supports representation of Python @@ -38,6 +114,22 @@ def unbounded_varchar(self): return exclusions.closed() + @property + def temp_table_reflection(self): + """Target database must support reflection of a temporary table whose + definition includes a primary key. + + Db2 "created global temporary tables" (CGTT) cannot define a PRIMARY + KEY, UNIQUE, FOREIGN KEY or index; attempting it raises SQL0526N + ("functionality that cannot be used with temporary tables"). The + suite's temp-table fixture hard-codes ``primary_key=True``, so Db2 + cannot create the table in the required shape. Declaring this closed + skips the temp-table reflection tests (and, via inheritance, + ``temp_table_reflect_indexes``) instead of erroring on every one. + """ + + return exclusions.closed() + #@property #def offset(self): # return exclusions.closed() diff --git a/requirements-test.txt b/requirements-test.txt new file mode 100644 index 0000000..52a3be5 --- /dev/null +++ b/requirements-test.txt @@ -0,0 +1,16 @@ +# Test dependencies for the ibm_db_sa test suite. +# +# pip install -r requirements-test.txt +# +# The test suite runs on SQLAlchemy 1.4.x, 2.0.x and 2.1.x. To test a specific +# version, install it explicitly, for example: +# pip install "sqlalchemy==2.1.*" +# or the SQLAlchemy development branch: +# pip install "sqlalchemy @ git+https://github.com/sqlalchemy/sqlalchemy.git@main" +# +# See README.unittests.md for details. + +sqlalchemy>=1.4,<2.2 +pytest>=7.0 +# Required only for the live-database compliance suite against Db2: +ibm_db>=2.0.0 diff --git a/run_tests.py b/run_tests.py index 7e3cba7..8199c98 100644 --- a/run_tests.py +++ b/run_tests.py @@ -1,11 +1,21 @@ -from sqlalchemy.dialects import registry +#!/usr/bin/env python +"""Convenience wrapper to run the ibm_db_sa test suite via pytest. -registry.register("db2", "ibm_db_sa.ibm_db", "DB2Dialect_ibm_db") -registry.register("db2.ibm_db", "ibm_db_sa.ibm_db", "DB2Dialect_ibm_db") -registry.register("db2.pyodbc", "ibm_db_sa.pyodbc", "DB2Dialect_pyodbc") -registry.register("db2.pyodbc400", "ibm_db_sa.pyodbc", "AS400Dialect_pyodbc") +This replaces the old nose/`sqlalchemy.testing.runner` entry point. +It simply forwards all command-line arguments to pytest, so you can do:: -from sqlalchemy.testing import runner + python run_tests.py test/test_dialect.py -v + python run_tests.py --dburi "db2+ibm_db://user:pass@host:50000/DB" -runner.main() +Dialect registration and the SQLAlchemy pytest plugin are wired up by +``conftest.py`` at the repository root. +""" + +import sys + +import pytest + +if __name__ == "__main__": + args = sys.argv[1:] or ["test/test_dialect.py"] + raise SystemExit(pytest.main(args)) diff --git a/setup.cfg b/setup.cfg index 9504a37..e23f3b2 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,19 +1,31 @@ [egg_info] tag_build = -[nosetests] -with-sqla_testing = true -where = test -cover-package = ibm_db -with-coverage = 1 -cover-erase = 1 -verbosity = 1 +[tool:pytest] +addopts = --tb native -v -r fxX +python_files = test/*test_*.py +# Markers used by SQLAlchemy's test suite. +markers = + backend + sparse_backend + sparse_driver_backend + mypy + memory_intensive + timing_intensive [sqla_testing] -requirement_cls=ibm_db_sa.requirements:Requirements -profile_file=.profiles.txt +requirement_cls = ibm_db_sa.requirements:Requirements +profile_file = test/profiles.txt [db] -default=db2+ibm_db://db2inst2:db2inst2@/test -sqlite=sqlite:///:memory: - +# 'default' is used when neither --dburi nor --db is passed. Keep it as +# sqlite in-memory so the OFFLINE tests (test/test_dialect.py) run with no +# Db2 available. The named 'db2' entry below is selected only with +# '--db db2' (or override per-run with --dburi). +# +# NOTE: Do NOT commit real credentials here. Point 'db2' at your own Db2 +# instance locally, or prefer passing the URL per-run with +# pytest --dburi "ibm_db_sa://user:pass@host:port/DBNAME" test/ +default = sqlite:///:memory: +sqlite = sqlite:///:memory: +db2 = ibm_db_sa://user:pass@host:port/DBNAME diff --git a/setup.py b/setup.py index 6e6beb6..8b7a0b8 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ v.close() readme = os.path.join(os.path.dirname(__file__), 'README.md') -sqlalchemy_requirement = "sqlalchemy>=1.3.5,<2.1" +sqlalchemy_requirement = "sqlalchemy>=1.4,<2.2" if 'USE_PYODBC' in os.environ and os.environ['USE_PYODBC'] == '1': require = [sqlalchemy_requirement] else: @@ -38,7 +38,7 @@ long_description=open(readme).read(), platforms='All', install_requires= require, - python_requires=">=3.9", + python_requires=">=3.10", packages=['ibm_db_sa'], entry_points={ 'sqlalchemy.dialects': [ @@ -57,6 +57,8 @@ }, zip_safe=False, extras_require={ - "dev": ["nose>=0.11"] + "test": ["pytest>=7.0"], + # Deprecated alias of "test", kept so "ibm_db_sa[dev]" still installs. + "dev": ["pytest>=7.0"], } ) diff --git a/test/test_dialect.py b/test/test_dialect.py new file mode 100644 index 0000000..0b5d053 --- /dev/null +++ b/test/test_dialect.py @@ -0,0 +1,301 @@ +"""Offline unit tests for the ibm_db_sa dialect. + +These tests do not need a Db2 connection; they only compile SQL, DDL and types. +Run them with ``pytest test/test_dialect.py``. +""" + +import decimal + +import pytest +from sqlalchemy import ( + BigInteger, + Boolean, + Column, + Enum, + Float, + Integer, + MetaData, + Numeric, + String, + Table, + UnicodeText, + exc, + func, + literal, + literal_column, + select, +) +from sqlalchemy.sql import quoted_name +from sqlalchemy.testing import AssertsCompiledSQL, fixtures + +from ibm_db_sa.ibm_db import SA_VERSION_MM, DB2Dialect_ibm_db + + +def _dialect(dbms_name=None): + dialect = DB2Dialect_ibm_db() + dialect.dbms_name = dbms_name + return dialect + + +class _TableFixture: + def setup_method(self): + self.metadata = MetaData() + self.table = Table( + "some_table", + self.metadata, + Column("id", Integer, primary_key=True), + Column("name", String(50)), + Column("big", BigInteger), + ) + + +class CompileTest(_TableFixture, fixtures.TestBase, AssertsCompiledSQL): + __dialect__ = _dialect() + + def test_select_all(self): + self.assert_compile( + select(self.table.c.id, self.table.c.name), + "SELECT some_table.id, some_table.name FROM some_table", + ) + + def test_limit_renders(self): + self.assert_compile( + select(self.table.c.id).limit(5), + "SELECT some_table.id FROM some_table LIMIT __[POSTCOMPILE_param_1]", + ) + + def test_offset_uses_rownumber(self): + self.assert_compile( + select(self.table.c.id).offset(10), + '( SELECT "Z.__db2_1" FROM ( SELECT "Z.__db2_1", ' + '( ROW_NUMBER() OVER() ) AS "Z.__ROWNUM" FROM ( ' + 'SELECT some_table.id AS "Z.__db2_1" FROM some_table ) AS M ) Z ' + 'WHERE "Z.__ROWNUM" > __[POSTCOMPILE_param_1] )', + ) + + def test_offset_in_subquery_is_not_truncated(self): + # Only the trailing OFFSET of each SELECT is rewritten; the outer + # WHERE clause after the subquery must be kept intact. + sub = select(self.table.c.id).offset(3).subquery() + sql = str( + select(sub.c.id).where(sub.c.id > 1).compile(dialect=self.__dialect__) + ) + assert "OFFSET" not in sql.upper() + assert sql.rstrip().endswith("WHERE anon_1.id > ?") + + def test_expr_offset_is_fully_stripped(self): + stmt = select(self.table.c.id).offset(literal_column("1") + literal_column("2")) + sql = str(stmt.compile(dialect=self.__dialect__)) + assert "OFFSET" not in sql.upper() + assert '"Z.__ROWNUM" > 1 + 2' in sql + + def test_outer_expr_offset_with_subquery_offset(self): + sub = select(self.table.c.id).offset(3).subquery() + stmt = select(sub.c.id).offset(literal_column("1") + literal_column("2")) + sql = str(stmt.compile(dialect=self.__dialect__)) + assert "OFFSET" not in sql.upper() + assert sql.count("ROW_NUMBER()") == 2 + assert '"Z.__ROWNUM" > 1 + 2' in sql + + def test_avg_wraps_double(self): + self.assert_compile( + select(func.avg(self.table.c.id)), + "SELECT AVG(DOUBLE((some_table.id))) AS avg_1 FROM some_table", + ) + + def test_mod_operator(self): + self.assert_compile( + select(self.table.c.id % 3), + "SELECT mod(some_table.id, ?) AS anon_1 FROM some_table", + ) + + def test_truediv_casts_divisor(self): + if SA_VERSION_MM < (2, 0): + expected = "SELECT some_table.id / CAST(? AS INT) AS anon_1 FROM some_table" + else: + expected = ( + "SELECT some_table.id / CAST(CAST(? AS INT) AS DECIMAL(19, 6)) " + "AS anon_1 FROM some_table" + ) + self.assert_compile(select(self.table.c.id / 2), expected) + + @pytest.mark.skipif(SA_VERSION_MM < (2, 0), reason="'//' requires SQLAlchemy 2.0+") + def test_floordiv_integers(self): + self.assert_compile( + select(self.table.c.id // self.table.c.id), + "SELECT CAST(some_table.id AS INT) / CAST(some_table.id AS INT) " + "AS anon_1 FROM some_table", + ) + + def test_bigint_bind_is_cast(self): + self.assert_compile( + select(self.table.c.id).where(self.table.c.big == 5), + "SELECT some_table.id FROM some_table " + "WHERE some_table.big = CAST(? AS BIGINT)", + ) + + def test_integer_bind_is_not_cast(self): + self.assert_compile( + select(self.table.c.name).where(self.table.c.id == 5), + "SELECT some_table.name FROM some_table WHERE some_table.id = ?", + ) + + def test_non_ascii_literal_renders_ux_constant(self): + self.assert_compile( + select(literal("\u00e9")), + "SELECT UX'00E9' AS anon_1 FROM SYSIBM.SYSDUMMY1", + literal_binds=True, + ) + + def test_ascii_literal_unchanged(self): + self.assert_compile( + select(literal("abc")), + "SELECT 'abc' AS anon_1 FROM SYSIBM.SYSDUMMY1", + literal_binds=True, + ) + + +class AS400CompileTest(_TableFixture, fixtures.TestBase, AssertsCompiledSQL): + __dialect__ = _dialect("AS") + + def test_empty_in_uses_typed_null(self): + self.assert_compile( + select(self.table.c.id).where(self.table.c.id.in_([])), + "SELECT some_table.id FROM some_table " + "WHERE some_table.id IN (CAST(NULL AS INTEGER)) AND (1 != 1)", + render_postcompile=True, + ) + + def test_empty_not_in_uses_typed_null(self): + self.assert_compile( + select(self.table.c.id).where(self.table.c.id.not_in([])), + "SELECT some_table.id FROM some_table " + "WHERE (some_table.id NOT IN (CAST(NULL AS INTEGER)) OR (1 = 1))", + render_postcompile=True, + ) + + +class TypeCompileTest(fixtures.TestBase, AssertsCompiledSQL): + __dialect__ = _dialect() + + @pytest.mark.parametrize( + "type_, expected", + [ + (String(40), "VARCHAR(40)"), + (Integer(), "INT"), + (BigInteger(), "BIGINT"), + (Boolean(), "BOOLEAN"), + (UnicodeText(), "DBCLOB(1M)"), + (Enum("a", "bb", name="e"), "VARCHAR(2 CODEUNITS32)"), + ], + ) + def test_luw_types(self, type_, expected): + self.assert_compile(type_, expected) + + @pytest.mark.parametrize( + "type_, expected", + [ + (Boolean(), "SMALLINT"), + (UnicodeText(), "DBCLOB(1M) CCSID 1200"), + (Enum("a", "bb", name="e"), "VARCHAR(2) CCSID 1208"), + (String(40), "VARCHAR(40) CCSID 1208"), + ], + ) + def test_as400_types(self, type_, expected): + self.assert_compile(type_, expected, dialect=_dialect("AS")) + + def test_zos_enum_has_no_codeunits(self): + self.assert_compile( + Enum("a", "bb", name="e"), "VARCHAR(2)", dialect=_dialect("DB2") + ) + + def test_server_type_is_per_dialect(self): + # Two dialects in one process must not share the server type. + as400, luw = _dialect("AS"), _dialect("DB2/LINUXX8664") + assert as400.type_compiler.process(Boolean()) == "SMALLINT" + assert luw.type_compiler.process(Boolean()) == "BOOLEAN" + + +class CreateTableAsTest(fixtures.TestBase, AssertsCompiledSQL): + __dialect__ = _dialect() + + def _ctas(self, **kw): + schema = pytest.importorskip("sqlalchemy.schema") + if not hasattr(schema, "CreateTableAs"): + pytest.skip("CreateTableAs requires SQLAlchemy 2.1+") + src = Table("src", MetaData(), Column("id", Integer)) + return schema.CreateTableAs(select(src.c.id), "dst", **kw) + + def test_create_table_as(self): + self.assert_compile( + self._ctas(), + "CREATE TABLE dst AS (SELECT src.id FROM src) WITH DATA", + ) + + @pytest.mark.parametrize("option", ["temporary", "if_not_exists"]) + def test_unsupported_options_raise(self, option): + stmt = self._ctas(**{option: True}) + with pytest.raises(exc.CompileError): + stmt.compile(dialect=self.__dialect__) + + +class NameNormalizationTest(fixtures.TestBase): + def setup_method(self): + self.reflector = _dialect()._reflector + + @pytest.mark.parametrize( + "name, expected, quote", + [ + ("SOME_TABLE", "some_table", None), + ("REF", "ref", None), + ("MixedCase", "MixedCase", True), + ("lower", "lower", True), + ], + ) + def test_normalize_name(self, name, expected, quote): + result = self.reflector.normalize_name(name) + assert result == expected + assert getattr(result, "quote", None) is quote + + def test_normalize_none(self): + assert self.reflector.normalize_name(None) is None + + @pytest.mark.parametrize( + "name, expected", + [ + ("some_table", "SOME_TABLE"), + ("MixedCase", "MixedCase"), + (quoted_name("lower", quote=True), "lower"), + ], + ) + def test_denormalize_name(self, name, expected): + assert self.reflector.denormalize_name(name) == expected + + @pytest.mark.parametrize("name", ["SOME_TABLE", "MixedCase", "lower"]) + def test_round_trip(self, name): + r = self.reflector + assert r.denormalize_name(r.normalize_name(name)) == name + + +class HomogenizeNumericTest(fixtures.TestBase): + homogenize = staticmethod(DB2Dialect_ibm_db._homogenize_numeric_columns) + + def test_decimal_column_uses_decimal(self): + result = self.homogenize([(1,), (1.5,)], [Numeric(10, 2)]) + assert result == [(decimal.Decimal("1"),), (decimal.Decimal("1.5"),)] + + def test_float_column_uses_float(self): + result = self.homogenize([(1,), (1.5,)], [Float()]) + assert result == [(1.0,), (1.5,)] + assert all(isinstance(row[0], float) for row in result) + + def test_non_numeric_column_is_untouched(self): + params = [(1,), (1.5,)] + assert self.homogenize(params, [Integer()]) is params + + def test_uniform_column_is_untouched(self): + params = [(1, "a"), (2, "b")] + assert self.homogenize(params, [Numeric(), String()]) is params + + def test_unknown_types_fall_back_to_all_columns(self): + assert self.homogenize([(1,), (1.5,)]) == [(1.0,), (1.5,)] diff --git a/test/test_out_parameters.py b/test/test_out_parameters.py index b889e06..1f5d5d2 100644 --- a/test/test_out_parameters.py +++ b/test/test_out_parameters.py @@ -1,33 +1,51 @@ -from sqlalchemy.testing import fixtures, config -from sqlalchemy import text, bindparam, outparam -from sqlalchemy import Float, Integer, String +"""Live-database test for Db2 stored-procedure OUT parameters. + +This test only runs when a real Db2 backend is configured (``--dburi``), +because it creates and calls a stored procedure. It is skipped otherwise. + +Modernized for the SQLAlchemy 2.0 connection API (the old connectionless +``config.db.execute(...)`` and ``outparam()`` helpers were removed in 2.0). +""" + +from sqlalchemy import text +from sqlalchemy.testing import config, fixtures from sqlalchemy.testing.assertions import eq_ class OutParamTest(fixtures.TestBase): + __only_on__ = "db2" + __backend__ = True @classmethod - def setup_class(cls): - config.db.execute(""" - create or replace procedure foo(IN x_in integer, OUT x_out integer, OUT y_out integer, OUT z_out varchar(20)) - BEGIN + def setup_test_class(cls): + with config.db.begin() as conn: + conn.exec_driver_sql( + """ + CREATE OR REPLACE PROCEDURE foo( + IN x_in INTEGER, + OUT x_out INTEGER, + OUT y_out INTEGER, + OUT z_out VARCHAR(20)) + BEGIN SET x_out = 10; SET y_out = x_in * 15; SET z_out = NULL; - END - """) - - def test_out_params(self): - result = \ - config.db.execute(text('call foo(:x_in, :x_out, :y_out, ' - ':z_out)', - bindparams=[bindparam('x_in'), - outparam('x_out'), - outparam('y_out'), - outparam('z_out')]), x_in=5, x_out=0, y_out=0, z_out='') - eq_(result.out_parameters, {'x_out': 10, 'y_out': 75, 'z_out': None}) - assert isinstance(result.out_parameters['x_out'], long) + END + """ + ) + + def test_out_params(self, connection): + result = connection.execute( + text("call foo(:x_in, :x_out, :y_out, :z_out)"), + {"x_in": 5, "x_out": 0, "y_out": 0, "z_out": ""}, + ) + eq_( + result.out_parameters, + {"x_out": 10, "y_out": 75, "z_out": None}, + ) + assert isinstance(result.out_parameters["x_out"], int) @classmethod - def teardown_class(cls): - config.db.execute("DROP PROCEDURE foo") \ No newline at end of file + def teardown_test_class(cls): + with config.db.begin() as conn: + conn.exec_driver_sql("DROP PROCEDURE foo")