diff --git a/docs/changes/newsfragments/8634.improved b/docs/changes/newsfragments/8634.improved new file mode 100644 index 000000000000..3d78c2372f47 --- /dev/null +++ b/docs/changes/newsfragments/8634.improved @@ -0,0 +1,3 @@ +Loading data from a ``DataSet`` is now faster. Array and complex valued data is read +several times faster since the header of the stored arrays is only parsed once per +parameter rather than once per stored value. diff --git a/src/qcodes/dataset/sqlite/database.py b/src/qcodes/dataset/sqlite/database.py index cb39c378c09d..84ebb70b3181 100644 --- a/src/qcodes/dataset/sqlite/database.py +++ b/src/qcodes/dataset/sqlite/database.py @@ -6,13 +6,14 @@ from __future__ import annotations +import functools import io import math import sqlite3 import sys from contextlib import contextmanager from os.path import expanduser, normpath -from typing import TYPE_CHECKING, Literal +from typing import TYPE_CHECKING, Any, Literal import numpy as np import numpy.typing as npt @@ -52,16 +53,94 @@ def _adapt_array(arr: npt.NDArray) -> sqlite3.Binary: def _convert_array(text: bytes) -> npt.NDArray: - # Using np.lib.format.read_array (counterpart of np.lib.format.write_array) - # npy format version 3.0 is 3 times faster than previous verions (no clean up step - # for python 2 backward compatibility) - return np.lib.format.read_array(io.BytesIO(text), allow_pickle=False) + """ + Convert the ``.npy`` serialization of an array (as written by + :func:`_adapt_array`) back into an array. + + This is equivalent to :func:`numpy.lib.format.read_array` but caches the + parsing of the header, which otherwise dominates the time it takes to + load small and medium sized arrays. All rows of a given array parameter + typically share the same header. Input that cannot be handled by the fast + path is passed on to :func:`numpy.lib.format.read_array`. + """ + header_info = _npy_header_info(text) + if header_info is None: + return np.lib.format.read_array(io.BytesIO(text), allow_pickle=False) + offset, shape, fortran_order, dtype = header_info + count = math.prod(shape) + if len(text) - offset < count * dtype.itemsize: + return np.lib.format.read_array(io.BytesIO(text), allow_pickle=False) + # copy since frombuffer returns a read only view of the bytes + array = np.frombuffer(text, dtype=dtype, count=count, offset=offset).copy() + if fortran_order: + return array.reshape(shape[::-1]).transpose() + return array.reshape(shape) + + +_NPY_MAGIC_PREFIX = b"\x93NUMPY" +# The default ``max_header_size`` of numpy's readers. Larger headers are +# rejected by numpy, so they are passed on to it rather than cached. +_NPY_MAX_HEADER_SIZE = 10_000 + + +def _npy_header_info( + text: bytes, +) -> tuple[int, tuple[int, ...], bool, np.dtype[Any]] | None: + """ + Return the offset of the data, shape, fortran order and dtype of an + ``.npy`` serialized array or None if the array should be read with + :func:`numpy.lib.format.read_array`. + """ + if len(text) < 12 or not text.startswith(_NPY_MAGIC_PREFIX): + return None + major_version = text[6] + if major_version == 1: + header_size = int.from_bytes(text[8:10], "little") + offset = 10 + header_size + elif major_version in (2, 3): + header_size = int.from_bytes(text[8:12], "little") + offset = 12 + header_size + else: + return None + # Only cache complete headers of a bounded size so that malformed input + # can not fill the cache with arbitrarily large byte strings. + if header_size > _NPY_MAX_HEADER_SIZE or offset > len(text): + return None + header_info = _parse_npy_header(text[:offset]) + if header_info is None: + return None + return (offset, *header_info) + + +@functools.lru_cache(maxsize=128) +def _parse_npy_header( + header: bytes, +) -> tuple[tuple[int, ...], bool, np.dtype[Any]] | None: + # numpy's public header readers decode the header as latin1 while + # version 3.0 headers are utf8 encoded. These are only identical for + # ascii, which covers all headers written by QCoDeS. + header_text_start = 10 if header[6] == 1 else 12 + if not header[header_text_start:].isascii(): + return None + fp = io.BytesIO(header) + try: + major_version, _ = np.lib.format.read_magic(fp) + if major_version == 1: + shape, fortran_order, dtype = np.lib.format.read_array_header_1_0(fp) + else: + shape, fortran_order, dtype = np.lib.format.read_array_header_2_0(fp) + except ValueError: + return None + # The dtype is shared by all arrays read with this header. The field names + # of structured dtypes can be modified in place, which would affect all + # later reads, so those are left to numpy. + if dtype.hasobject or dtype.itemsize == 0 or dtype.names is not None: + return None + return shape, fortran_order, dtype def _convert_complex(text: bytes) -> np.complexfloating: - out = io.BytesIO(text) - out.seek(0) - return np.load(out)[0] + return _convert_array(text)[0] this_session_default_encoding = sys.getdefaultencoding() @@ -89,20 +168,10 @@ def _convert_numeric(value: bytes) -> float | int | str: # Let string casting fail if bytes encoding is invalid return str(value, encoding=this_session_default_encoding) - # If that worked, e.g. did not raise an exception, then we check if the outcome is - # either an infinity or a NaN - # For a single value, math.isfinite is 10 times faster than np.isinfinite (or - # combining np.isnan and np.isinf) - if not math.isfinite(numeric): - return numeric - - # If it is not 'nan' and not 'inf', then we need to see if the value is really an - # integer or with floating point digits - numeric_int = int(numeric) - if numeric != numeric_int: - return numeric - else: - return numeric_int + # is_integer is False for NaN and infinity so these are returned as floats + if numeric.is_integer(): + return int(numeric) + return numeric def _adapt_float(fl: float) -> float | str: diff --git a/tests/dataset/test_sqlite_converters.py b/tests/dataset/test_sqlite_converters.py new file mode 100644 index 000000000000..0ed411fda3fa --- /dev/null +++ b/tests/dataset/test_sqlite_converters.py @@ -0,0 +1,210 @@ +from __future__ import annotations + +import io +import math +from typing import TYPE_CHECKING, Any, Literal + +import numpy as np +import pytest + +from qcodes.dataset.sqlite.database import ( + _adapt_array, + _adapt_complex, + _convert_array, + _convert_complex, + _convert_numeric, + _parse_npy_header, +) + +if TYPE_CHECKING: + import numpy.typing as npt + from pytest_mock import MockerFixture + + +def _serialize( + array: npt.NDArray, version: tuple[int, int] | None, allow_pickle: bool = False +) -> bytes: + out = io.BytesIO() + if version is None: + np.save(out, array, allow_pickle=allow_pickle) + else: + np.lib.format.write_array( + out, array, version=version, allow_pickle=allow_pickle + ) + return out.getvalue() + + +def _read_with_numpy(serialized: bytes) -> npt.NDArray: + return np.lib.format.read_array(io.BytesIO(serialized), allow_pickle=False) + + +@pytest.mark.parametrize("version", [(1, 0), (2, 0), (3, 0), None]) +@pytest.mark.parametrize("order", ["C", "F"]) +@pytest.mark.parametrize("shape", [(), (0,), (1,), (7,), (3, 4), (2, 3, 4), (0, 3)]) +@pytest.mark.parametrize( + "dtype", + [ + "f8", + ">f8", + "f4", + "i1", + ">i4", + "u8", + "c8", + "c16", + "?", + "U3", + "S4", + "M8[ns]", + ], +) +def test_convert_array_matches_numpy( + mocker: MockerFixture, + dtype: Any, + shape: tuple[int, ...], + order: Literal["C", "F"], + version: tuple[int, int] | None, +) -> None: + array = np.asarray(np.zeros(shape, dtype=np.dtype(dtype)), order=order) + array.flat = np.arange(array.size) % 100 + serialized = _serialize(array, version) + expected = _read_with_numpy(serialized) + + read_array_spy = mocker.spy(np.lib.format, "read_array") + result = _convert_array(serialized) + + read_array_spy.assert_not_called() + assert result.dtype == expected.dtype + assert result.shape == expected.shape + assert result.flags.c_contiguous == expected.flags.c_contiguous + assert result.flags.f_contiguous == expected.flags.f_contiguous + assert result.flags.writeable + assert result.tobytes() == expected.tobytes() + + +def test_convert_array_round_trips_adapted_array() -> None: + array = np.random.default_rng().random((3, 5)) + result = _convert_array(bytes(_adapt_array(array))) + np.testing.assert_array_equal(result, array) + # the returned array must not share memory with the immutable input bytes + result[0, 0] = 1.0 + + +def test_convert_array_non_ascii_header_falls_back_to_numpy( + mocker: MockerFixture, +) -> None: + serialized = _serialize(np.zeros(2, dtype=[("\u00e5", "f8")]), (3, 0)) + read_array_spy = mocker.spy(np.lib.format, "read_array") + + result = _convert_array(serialized) + + read_array_spy.assert_called_once() + assert result.dtype == _read_with_numpy(serialized).dtype + + +@pytest.mark.parametrize( + "dtype", + [ + [("a", "f8"), ("b", "i4")], + [("a", "f8"), ("b", [("c", "i4"), ("d", "f4")])], + ], +) +def test_convert_array_structured_dtype_is_not_shared( + mocker: MockerFixture, dtype: Any +) -> None: + serialized = _serialize(np.zeros(2, dtype=dtype), (3, 0)) + expected = _read_with_numpy(serialized) + read_array_spy = mocker.spy(np.lib.format, "read_array") + + result = _convert_array(serialized) + read_array_spy.assert_called_once() + assert result.dtype == expected.dtype + # renaming the fields of one result must not affect later reads + assert result.dtype.names is not None + result.dtype.names = tuple(f"renamed_{name}" for name in result.dtype.names) + nested_dtype = result.dtype[-1] + if nested_dtype.names is not None: + nested_dtype.names = tuple(f"renamed_{name}" for name in nested_dtype.names) + + assert _convert_array(serialized).dtype == expected.dtype + + +@pytest.mark.parametrize( + ("header_size", "available_header_bytes"), + [(10_001, 10_001), (5_000, 100), (2**31, 100)], + ids=["too_large", "truncated", "too_large_and_truncated"], +) +def test_convert_array_invalid_header_size_is_not_cached( + header_size: int, available_header_bytes: int +) -> None: + serialized = ( + b"\x93NUMPY\x03\x00" + + header_size.to_bytes(4, "little") + + b" " * available_header_bytes + ) + _parse_npy_header.cache_clear() + + with pytest.raises(ValueError): + _convert_array(serialized) + + assert _parse_npy_header.cache_info().currsize == 0 + + +def test_convert_array_object_array_raises() -> None: + serialized = _serialize(np.array([1, "a"], dtype=object), (3, 0), True) + with pytest.raises(ValueError, match="Object arrays cannot be loaded"): + _convert_array(serialized) + + +@pytest.mark.parametrize("serialized", [b"", b"not an array", b"\x93NUMP"]) +def test_convert_array_invalid_input_raises(serialized: bytes) -> None: + with pytest.raises(ValueError): + _convert_array(serialized) + + +def test_convert_array_truncated_data_raises() -> None: + serialized = bytes(_adapt_array(np.arange(10.0)))[:-8] + with pytest.raises(ValueError, match="array data"): + _convert_array(serialized) + + +@pytest.mark.parametrize( + "value", + [1 + 2j, np.complex64(3 - 1j), np.complex128(-0.5 + 0.25j), complex("nan+infj")], +) +def test_convert_complex(value: complex | np.complexfloating) -> None: + serialized = bytes(_adapt_complex(value)) + expected = np.load(io.BytesIO(serialized))[0] + + result = _convert_complex(serialized) + + assert type(result) is type(expected) + np.testing.assert_array_equal(result, expected) + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + (b"1.5", 1.5), + (b"-2.25e-3", -2.25e-3), + (b"3", 3), + (b"3.0", 3), + (b"-0.0", 0), + (b"1e20", 10**20), + (b"inf", math.inf), + (b"-Infinity", -math.inf), + (b"some text", "some text"), + ], +) +def test_convert_numeric(value: bytes, expected: float | str) -> None: + result = _convert_numeric(value) + assert type(result) is type(expected) + assert result == expected + + +@pytest.mark.parametrize("value", [b"nan", b"NaN", b"-nan"]) +def test_convert_numeric_nan(value: bytes) -> None: + result = _convert_numeric(value) + assert isinstance(result, float) + assert math.isnan(result)