From ccd84af6b136d9aab2b6feb30f371f4e2d8e12c8 Mon Sep 17 00:00:00 2001 From: "Jens H. Nielsen" Date: Sat, 17 Oct 2026 09:24:26 +0200 Subject: [PATCH 1/2] Speed up reading array and complex data from sqlite Parsing the npy header of every stored value dominated the time it takes to load array and complex data. Cache the parsed header, since all rows of a parameter share it, and fall back to numpy's read_array for anything the fast path does not handle. --- docs/changes/newsfragments/8634.improved | 3 + src/qcodes/dataset/sqlite/database.py | 95 +++++++++++- tests/dataset/test_sqlite_converters.py | 182 +++++++++++++++++++++++ 3 files changed, 272 insertions(+), 8 deletions(-) create mode 100644 docs/changes/newsfragments/8634.improved create mode 100644 tests/dataset/test_sqlite_converters.py diff --git a/docs/changes/newsfragments/8634.improved b/docs/changes/newsfragments/8634.improved new file mode 100644 index 000000000000..3d78c2372f47 --- /dev/null +++ b/docs/changes/newsfragments/8634.improved @@ -0,0 +1,3 @@ +Loading data from a ``DataSet`` is now faster. Array and complex valued data is read +several times faster since the header of the stored arrays is only parsed once per +parameter rather than once per stored value. diff --git a/src/qcodes/dataset/sqlite/database.py b/src/qcodes/dataset/sqlite/database.py index cb39c378c09d..d2f4612266a5 100644 --- a/src/qcodes/dataset/sqlite/database.py +++ b/src/qcodes/dataset/sqlite/database.py @@ -6,13 +6,14 @@ from __future__ import annotations +import functools import io import math import sqlite3 import sys from contextlib import contextmanager from os.path import expanduser, normpath -from typing import TYPE_CHECKING, Literal +from typing import TYPE_CHECKING, Any, Literal import numpy as np import numpy.typing as npt @@ -52,16 +53,94 @@ def _adapt_array(arr: npt.NDArray) -> sqlite3.Binary: def _convert_array(text: bytes) -> npt.NDArray: - # Using np.lib.format.read_array (counterpart of np.lib.format.write_array) - # npy format version 3.0 is 3 times faster than previous verions (no clean up step - # for python 2 backward compatibility) - return np.lib.format.read_array(io.BytesIO(text), allow_pickle=False) + """ + Convert the ``.npy`` serialization of an array (as written by + :func:`_adapt_array`) back into an array. + + This is equivalent to :func:`numpy.lib.format.read_array` but caches the + parsing of the header, which otherwise dominates the time it takes to + load small and medium sized arrays. All rows of a given array parameter + typically share the same header. Input that cannot be handled by the fast + path is passed on to :func:`numpy.lib.format.read_array`. + """ + header_info = _npy_header_info(text) + if header_info is None: + return np.lib.format.read_array(io.BytesIO(text), allow_pickle=False) + offset, shape, fortran_order, dtype = header_info + count = math.prod(shape) + if len(text) - offset < count * dtype.itemsize: + return np.lib.format.read_array(io.BytesIO(text), allow_pickle=False) + # copy since frombuffer returns a read only view of the bytes + array = np.frombuffer(text, dtype=dtype, count=count, offset=offset).copy() + if fortran_order: + return array.reshape(shape[::-1]).transpose() + return array.reshape(shape) + + +_NPY_MAGIC_PREFIX = b"\x93NUMPY" +# The default ``max_header_size`` of numpy's readers. Larger headers are +# rejected by numpy, so they are passed on to it rather than cached. +_NPY_MAX_HEADER_SIZE = 10_000 + + +def _npy_header_info( + text: bytes, +) -> tuple[int, tuple[int, ...], bool, np.dtype[Any]] | None: + """ + Return the offset of the data, shape, fortran order and dtype of an + ``.npy`` serialized array or None if the array should be read with + :func:`numpy.lib.format.read_array`. + """ + if len(text) < 12 or not text.startswith(_NPY_MAGIC_PREFIX): + return None + major_version = text[6] + if major_version == 1: + header_size = int.from_bytes(text[8:10], "little") + offset = 10 + header_size + elif major_version in (2, 3): + header_size = int.from_bytes(text[8:12], "little") + offset = 12 + header_size + else: + return None + # Only cache complete headers of a bounded size so that malformed input + # can not fill the cache with arbitrarily large byte strings. + if header_size > _NPY_MAX_HEADER_SIZE or offset > len(text): + return None + header_info = _parse_npy_header(text[:offset]) + if header_info is None: + return None + return (offset, *header_info) + + +@functools.lru_cache(maxsize=128) +def _parse_npy_header( + header: bytes, +) -> tuple[tuple[int, ...], bool, np.dtype[Any]] | None: + # numpy's public header readers decode the header as latin1 while + # version 3.0 headers are utf8 encoded. These are only identical for + # ascii, which covers all headers written by QCoDeS. + header_text_start = 10 if header[6] == 1 else 12 + if not header[header_text_start:].isascii(): + return None + fp = io.BytesIO(header) + try: + major_version, _ = np.lib.format.read_magic(fp) + if major_version == 1: + shape, fortran_order, dtype = np.lib.format.read_array_header_1_0(fp) + else: + shape, fortran_order, dtype = np.lib.format.read_array_header_2_0(fp) + except ValueError: + return None + # The dtype is shared by all arrays read with this header. The field names + # of structured dtypes can be modified in place, which would affect all + # later reads, so those are left to numpy. + if dtype.hasobject or dtype.itemsize == 0 or dtype.names is not None: + return None + return shape, fortran_order, dtype def _convert_complex(text: bytes) -> np.complexfloating: - out = io.BytesIO(text) - out.seek(0) - return np.load(out)[0] + return _convert_array(text)[0] this_session_default_encoding = sys.getdefaultencoding() diff --git a/tests/dataset/test_sqlite_converters.py b/tests/dataset/test_sqlite_converters.py new file mode 100644 index 000000000000..ed37f9746d93 --- /dev/null +++ b/tests/dataset/test_sqlite_converters.py @@ -0,0 +1,182 @@ +from __future__ import annotations + +import io +from typing import TYPE_CHECKING, Any, Literal + +import numpy as np +import pytest + +from qcodes.dataset.sqlite.database import ( + _adapt_array, + _adapt_complex, + _convert_array, + _convert_complex, + _parse_npy_header, +) + +if TYPE_CHECKING: + import numpy.typing as npt + from pytest_mock import MockerFixture + + +def _serialize( + array: npt.NDArray, version: tuple[int, int] | None, allow_pickle: bool = False +) -> bytes: + out = io.BytesIO() + if version is None: + np.save(out, array, allow_pickle=allow_pickle) + else: + np.lib.format.write_array( + out, array, version=version, allow_pickle=allow_pickle + ) + return out.getvalue() + + +def _read_with_numpy(serialized: bytes) -> npt.NDArray: + return np.lib.format.read_array(io.BytesIO(serialized), allow_pickle=False) + + +@pytest.mark.parametrize("version", [(1, 0), (2, 0), (3, 0), None]) +@pytest.mark.parametrize("order", ["C", "F"]) +@pytest.mark.parametrize("shape", [(), (0,), (1,), (7,), (3, 4), (2, 3, 4), (0, 3)]) +@pytest.mark.parametrize( + "dtype", + [ + "f8", + ">f8", + "f4", + "i1", + ">i4", + "u8", + "c8", + "c16", + "?", + "U3", + "S4", + "M8[ns]", + ], +) +def test_convert_array_matches_numpy( + mocker: MockerFixture, + dtype: Any, + shape: tuple[int, ...], + order: Literal["C", "F"], + version: tuple[int, int] | None, +) -> None: + array = np.asarray(np.zeros(shape, dtype=np.dtype(dtype)), order=order) + array.flat = np.arange(array.size) % 100 + serialized = _serialize(array, version) + expected = _read_with_numpy(serialized) + + read_array_spy = mocker.spy(np.lib.format, "read_array") + result = _convert_array(serialized) + + read_array_spy.assert_not_called() + assert result.dtype == expected.dtype + assert result.shape == expected.shape + assert result.flags.c_contiguous == expected.flags.c_contiguous + assert result.flags.f_contiguous == expected.flags.f_contiguous + assert result.flags.writeable + assert result.tobytes() == expected.tobytes() + + +def test_convert_array_round_trips_adapted_array() -> None: + array = np.random.default_rng().random((3, 5)) + result = _convert_array(bytes(_adapt_array(array))) + np.testing.assert_array_equal(result, array) + # the returned array must not share memory with the immutable input bytes + result[0, 0] = 1.0 + + +def test_convert_array_non_ascii_header_falls_back_to_numpy( + mocker: MockerFixture, +) -> None: + serialized = _serialize(np.zeros(2, dtype=[("\u00e5", "f8")]), (3, 0)) + read_array_spy = mocker.spy(np.lib.format, "read_array") + + result = _convert_array(serialized) + + read_array_spy.assert_called_once() + assert result.dtype == _read_with_numpy(serialized).dtype + + +@pytest.mark.parametrize( + "dtype", + [ + [("a", "f8"), ("b", "i4")], + [("a", "f8"), ("b", [("c", "i4"), ("d", "f4")])], + ], +) +def test_convert_array_structured_dtype_is_not_shared( + mocker: MockerFixture, dtype: Any +) -> None: + serialized = _serialize(np.zeros(2, dtype=dtype), (3, 0)) + expected = _read_with_numpy(serialized) + read_array_spy = mocker.spy(np.lib.format, "read_array") + + result = _convert_array(serialized) + read_array_spy.assert_called_once() + assert result.dtype == expected.dtype + # renaming the fields of one result must not affect later reads + assert result.dtype.names is not None + result.dtype.names = tuple(f"renamed_{name}" for name in result.dtype.names) + nested_dtype = result.dtype[-1] + if nested_dtype.names is not None: + nested_dtype.names = tuple(f"renamed_{name}" for name in nested_dtype.names) + + assert _convert_array(serialized).dtype == expected.dtype + + +@pytest.mark.parametrize( + ("header_size", "available_header_bytes"), + [(10_001, 10_001), (5_000, 100), (2**31, 100)], + ids=["too_large", "truncated", "too_large_and_truncated"], +) +def test_convert_array_invalid_header_size_is_not_cached( + header_size: int, available_header_bytes: int +) -> None: + serialized = ( + b"\x93NUMPY\x03\x00" + + header_size.to_bytes(4, "little") + + b" " * available_header_bytes + ) + _parse_npy_header.cache_clear() + + with pytest.raises(ValueError): + _convert_array(serialized) + + assert _parse_npy_header.cache_info().currsize == 0 + + +def test_convert_array_object_array_raises() -> None: + serialized = _serialize(np.array([1, "a"], dtype=object), (3, 0), True) + with pytest.raises(ValueError, match="Object arrays cannot be loaded"): + _convert_array(serialized) + + +@pytest.mark.parametrize("serialized", [b"", b"not an array", b"\x93NUMP"]) +def test_convert_array_invalid_input_raises(serialized: bytes) -> None: + with pytest.raises(ValueError): + _convert_array(serialized) + + +def test_convert_array_truncated_data_raises() -> None: + serialized = bytes(_adapt_array(np.arange(10.0)))[:-8] + with pytest.raises(ValueError, match="array data"): + _convert_array(serialized) + + +@pytest.mark.parametrize( + "value", + [1 + 2j, np.complex64(3 - 1j), np.complex128(-0.5 + 0.25j), complex("nan+infj")], +) +def test_convert_complex(value: complex | np.complexfloating) -> None: + serialized = bytes(_adapt_complex(value)) + expected = np.load(io.BytesIO(serialized))[0] + + result = _convert_complex(serialized) + + assert type(result) is type(expected) + np.testing.assert_array_equal(result, expected) + From c3323a3e4877b18e49809d62b3e17a1326f2bd38 Mon Sep 17 00:00:00 2001 From: "Jens H. Nielsen" Date: Sat, 17 Oct 2026 09:24:26 +0200 Subject: [PATCH 2/2] Speed up converting numeric values read from sqlite Use float.is_integer, which is False for NaN and infinity, rather than separately checking for finite values and comparing with the value converted to int. --- src/qcodes/dataset/sqlite/database.py | 18 ++++------------ tests/dataset/test_sqlite_converters.py | 28 +++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 14 deletions(-) diff --git a/src/qcodes/dataset/sqlite/database.py b/src/qcodes/dataset/sqlite/database.py index d2f4612266a5..84ebb70b3181 100644 --- a/src/qcodes/dataset/sqlite/database.py +++ b/src/qcodes/dataset/sqlite/database.py @@ -168,20 +168,10 @@ def _convert_numeric(value: bytes) -> float | int | str: # Let string casting fail if bytes encoding is invalid return str(value, encoding=this_session_default_encoding) - # If that worked, e.g. did not raise an exception, then we check if the outcome is - # either an infinity or a NaN - # For a single value, math.isfinite is 10 times faster than np.isinfinite (or - # combining np.isnan and np.isinf) - if not math.isfinite(numeric): - return numeric - - # If it is not 'nan' and not 'inf', then we need to see if the value is really an - # integer or with floating point digits - numeric_int = int(numeric) - if numeric != numeric_int: - return numeric - else: - return numeric_int + # is_integer is False for NaN and infinity so these are returned as floats + if numeric.is_integer(): + return int(numeric) + return numeric def _adapt_float(fl: float) -> float | str: diff --git a/tests/dataset/test_sqlite_converters.py b/tests/dataset/test_sqlite_converters.py index ed37f9746d93..0ed411fda3fa 100644 --- a/tests/dataset/test_sqlite_converters.py +++ b/tests/dataset/test_sqlite_converters.py @@ -1,6 +1,7 @@ from __future__ import annotations import io +import math from typing import TYPE_CHECKING, Any, Literal import numpy as np @@ -11,6 +12,7 @@ _adapt_complex, _convert_array, _convert_complex, + _convert_numeric, _parse_npy_header, ) @@ -180,3 +182,29 @@ def test_convert_complex(value: complex | np.complexfloating) -> None: assert type(result) is type(expected) np.testing.assert_array_equal(result, expected) + +@pytest.mark.parametrize( + ("value", "expected"), + [ + (b"1.5", 1.5), + (b"-2.25e-3", -2.25e-3), + (b"3", 3), + (b"3.0", 3), + (b"-0.0", 0), + (b"1e20", 10**20), + (b"inf", math.inf), + (b"-Infinity", -math.inf), + (b"some text", "some text"), + ], +) +def test_convert_numeric(value: bytes, expected: float | str) -> None: + result = _convert_numeric(value) + assert type(result) is type(expected) + assert result == expected + + +@pytest.mark.parametrize("value", [b"nan", b"NaN", b"-nan"]) +def test_convert_numeric_nan(value: bytes) -> None: + result = _convert_numeric(value) + assert isinstance(result, float) + assert math.isnan(result)