Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions docs/changes/newsfragments/8634.improved
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
Loading data from a ``DataSet`` is now faster. Array and complex valued data is read
several times faster since the header of the stored arrays is only parsed once per
parameter rather than once per stored value.
113 changes: 91 additions & 22 deletions src/qcodes/dataset/sqlite/database.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,14 @@

from __future__ import annotations

import functools
import io
import math
import sqlite3
import sys
from contextlib import contextmanager
from os.path import expanduser, normpath
from typing import TYPE_CHECKING, Literal
from typing import TYPE_CHECKING, Any, Literal

import numpy as np
import numpy.typing as npt
Expand Down Expand Up @@ -52,16 +53,94 @@ def _adapt_array(arr: npt.NDArray) -> sqlite3.Binary:


def _convert_array(text: bytes) -> npt.NDArray:
# Using np.lib.format.read_array (counterpart of np.lib.format.write_array)
# npy format version 3.0 is 3 times faster than previous verions (no clean up step
# for python 2 backward compatibility)
return np.lib.format.read_array(io.BytesIO(text), allow_pickle=False)
"""
Convert the ``.npy`` serialization of an array (as written by
:func:`_adapt_array`) back into an array.

This is equivalent to :func:`numpy.lib.format.read_array` but caches the
parsing of the header, which otherwise dominates the time it takes to
load small and medium sized arrays. All rows of a given array parameter
typically share the same header. Input that cannot be handled by the fast
path is passed on to :func:`numpy.lib.format.read_array`.
"""
header_info = _npy_header_info(text)
if header_info is None:
return np.lib.format.read_array(io.BytesIO(text), allow_pickle=False)
offset, shape, fortran_order, dtype = header_info
count = math.prod(shape)
if len(text) - offset < count * dtype.itemsize:
return np.lib.format.read_array(io.BytesIO(text), allow_pickle=False)
# copy since frombuffer returns a read only view of the bytes
array = np.frombuffer(text, dtype=dtype, count=count, offset=offset).copy()
Comment thread
Copilot marked this conversation as resolved.
Resolved
if fortran_order:
return array.reshape(shape[::-1]).transpose()
return array.reshape(shape)


_NPY_MAGIC_PREFIX = b"\x93NUMPY"
# The default ``max_header_size`` of numpy's readers. Larger headers are
# rejected by numpy, so they are passed on to it rather than cached.
_NPY_MAX_HEADER_SIZE = 10_000


def _npy_header_info(
text: bytes,
) -> tuple[int, tuple[int, ...], bool, np.dtype[Any]] | None:
"""
Return the offset of the data, shape, fortran order and dtype of an
``.npy`` serialized array or None if the array should be read with
:func:`numpy.lib.format.read_array`.
"""
if len(text) < 12 or not text.startswith(_NPY_MAGIC_PREFIX):
return None
major_version = text[6]
if major_version == 1:
header_size = int.from_bytes(text[8:10], "little")
offset = 10 + header_size
elif major_version in (2, 3):
header_size = int.from_bytes(text[8:12], "little")
offset = 12 + header_size
else:
return None
# Only cache complete headers of a bounded size so that malformed input
# can not fill the cache with arbitrarily large byte strings.
if header_size > _NPY_MAX_HEADER_SIZE or offset > len(text):
return None
header_info = _parse_npy_header(text[:offset])
if header_info is None:
return None
return (offset, *header_info)


@functools.lru_cache(maxsize=128)
def _parse_npy_header(
header: bytes,
) -> tuple[tuple[int, ...], bool, np.dtype[Any]] | None:
# numpy's public header readers decode the header as latin1 while
# version 3.0 headers are utf8 encoded. These are only identical for
# ascii, which covers all headers written by QCoDeS.
header_text_start = 10 if header[6] == 1 else 12
if not header[header_text_start:].isascii():
return None
fp = io.BytesIO(header)
try:
major_version, _ = np.lib.format.read_magic(fp)
if major_version == 1:
shape, fortran_order, dtype = np.lib.format.read_array_header_1_0(fp)
else:
shape, fortran_order, dtype = np.lib.format.read_array_header_2_0(fp)
except ValueError:
return None
# The dtype is shared by all arrays read with this header. The field names
# of structured dtypes can be modified in place, which would affect all
# later reads, so those are left to numpy.
if dtype.hasobject or dtype.itemsize == 0 or dtype.names is not None:
return None
return shape, fortran_order, dtype


def _convert_complex(text: bytes) -> np.complexfloating:
out = io.BytesIO(text)
out.seek(0)
return np.load(out)[0]
return _convert_array(text)[0]


this_session_default_encoding = sys.getdefaultencoding()
Expand Down Expand Up @@ -89,20 +168,10 @@ def _convert_numeric(value: bytes) -> float | int | str:
# Let string casting fail if bytes encoding is invalid
return str(value, encoding=this_session_default_encoding)

# If that worked, e.g. did not raise an exception, then we check if the outcome is
# either an infinity or a NaN
# For a single value, math.isfinite is 10 times faster than np.isinfinite (or
# combining np.isnan and np.isinf)
if not math.isfinite(numeric):
return numeric

# If it is not 'nan' and not 'inf', then we need to see if the value is really an
# integer or with floating point digits
numeric_int = int(numeric)
if numeric != numeric_int:
return numeric
else:
return numeric_int
# is_integer is False for NaN and infinity so these are returned as floats
if numeric.is_integer():
return int(numeric)
return numeric


def _adapt_float(fl: float) -> float | str:
Expand Down
210 changes: 210 additions & 0 deletions tests/dataset/test_sqlite_converters.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,210 @@
from __future__ import annotations

import io
import math
from typing import TYPE_CHECKING, Any, Literal

import numpy as np
import pytest

from qcodes.dataset.sqlite.database import (
_adapt_array,
_adapt_complex,
_convert_array,
_convert_complex,
_convert_numeric,
_parse_npy_header,
)

if TYPE_CHECKING:
import numpy.typing as npt
from pytest_mock import MockerFixture


def _serialize(
array: npt.NDArray, version: tuple[int, int] | None, allow_pickle: bool = False
) -> bytes:
out = io.BytesIO()
if version is None:
np.save(out, array, allow_pickle=allow_pickle)
else:
np.lib.format.write_array(
out, array, version=version, allow_pickle=allow_pickle
)
return out.getvalue()


def _read_with_numpy(serialized: bytes) -> npt.NDArray:
return np.lib.format.read_array(io.BytesIO(serialized), allow_pickle=False)


@pytest.mark.parametrize("version", [(1, 0), (2, 0), (3, 0), None])
@pytest.mark.parametrize("order", ["C", "F"])
@pytest.mark.parametrize("shape", [(), (0,), (1,), (7,), (3, 4), (2, 3, 4), (0, 3)])
@pytest.mark.parametrize(
"dtype",
[
"f8",
">f8",
"f4",
"i1",
">i4",
"u8",
"c8",
"c16",
"?",
"<U5",
">U3",
"S4",
"M8[ns]",
],
)
def test_convert_array_matches_numpy(
mocker: MockerFixture,
dtype: Any,
shape: tuple[int, ...],
order: Literal["C", "F"],
version: tuple[int, int] | None,
) -> None:
array = np.asarray(np.zeros(shape, dtype=np.dtype(dtype)), order=order)
array.flat = np.arange(array.size) % 100
serialized = _serialize(array, version)
expected = _read_with_numpy(serialized)

read_array_spy = mocker.spy(np.lib.format, "read_array")
result = _convert_array(serialized)

read_array_spy.assert_not_called()
assert result.dtype == expected.dtype
assert result.shape == expected.shape
assert result.flags.c_contiguous == expected.flags.c_contiguous
assert result.flags.f_contiguous == expected.flags.f_contiguous
assert result.flags.writeable
assert result.tobytes() == expected.tobytes()


def test_convert_array_round_trips_adapted_array() -> None:
array = np.random.default_rng().random((3, 5))
result = _convert_array(bytes(_adapt_array(array)))
np.testing.assert_array_equal(result, array)
# the returned array must not share memory with the immutable input bytes
result[0, 0] = 1.0


def test_convert_array_non_ascii_header_falls_back_to_numpy(
mocker: MockerFixture,
) -> None:
serialized = _serialize(np.zeros(2, dtype=[("\u00e5", "f8")]), (3, 0))
read_array_spy = mocker.spy(np.lib.format, "read_array")

result = _convert_array(serialized)

read_array_spy.assert_called_once()
assert result.dtype == _read_with_numpy(serialized).dtype


@pytest.mark.parametrize(
"dtype",
[
[("a", "f8"), ("b", "i4")],
[("a", "f8"), ("b", [("c", "i4"), ("d", "f4")])],
],
)
def test_convert_array_structured_dtype_is_not_shared(
mocker: MockerFixture, dtype: Any
) -> None:
serialized = _serialize(np.zeros(2, dtype=dtype), (3, 0))
expected = _read_with_numpy(serialized)
read_array_spy = mocker.spy(np.lib.format, "read_array")

result = _convert_array(serialized)
read_array_spy.assert_called_once()
assert result.dtype == expected.dtype
# renaming the fields of one result must not affect later reads
assert result.dtype.names is not None
result.dtype.names = tuple(f"renamed_{name}" for name in result.dtype.names)
nested_dtype = result.dtype[-1]
if nested_dtype.names is not None:
nested_dtype.names = tuple(f"renamed_{name}" for name in nested_dtype.names)

assert _convert_array(serialized).dtype == expected.dtype


@pytest.mark.parametrize(
("header_size", "available_header_bytes"),
[(10_001, 10_001), (5_000, 100), (2**31, 100)],
ids=["too_large", "truncated", "too_large_and_truncated"],
)
def test_convert_array_invalid_header_size_is_not_cached(
header_size: int, available_header_bytes: int
) -> None:
serialized = (
b"\x93NUMPY\x03\x00"
+ header_size.to_bytes(4, "little")
+ b" " * available_header_bytes
)
_parse_npy_header.cache_clear()

with pytest.raises(ValueError):
_convert_array(serialized)

assert _parse_npy_header.cache_info().currsize == 0


def test_convert_array_object_array_raises() -> None:
serialized = _serialize(np.array([1, "a"], dtype=object), (3, 0), True)
with pytest.raises(ValueError, match="Object arrays cannot be loaded"):
_convert_array(serialized)


@pytest.mark.parametrize("serialized", [b"", b"not an array", b"\x93NUMP"])
def test_convert_array_invalid_input_raises(serialized: bytes) -> None:
with pytest.raises(ValueError):
_convert_array(serialized)


def test_convert_array_truncated_data_raises() -> None:
serialized = bytes(_adapt_array(np.arange(10.0)))[:-8]
with pytest.raises(ValueError, match="array data"):
_convert_array(serialized)


@pytest.mark.parametrize(
"value",
[1 + 2j, np.complex64(3 - 1j), np.complex128(-0.5 + 0.25j), complex("nan+infj")],
)
def test_convert_complex(value: complex | np.complexfloating) -> None:
serialized = bytes(_adapt_complex(value))
expected = np.load(io.BytesIO(serialized))[0]

result = _convert_complex(serialized)

assert type(result) is type(expected)
np.testing.assert_array_equal(result, expected)


@pytest.mark.parametrize(
("value", "expected"),
[
(b"1.5", 1.5),
(b"-2.25e-3", -2.25e-3),
(b"3", 3),
(b"3.0", 3),
(b"-0.0", 0),
(b"1e20", 10**20),
(b"inf", math.inf),
(b"-Infinity", -math.inf),
(b"some text", "some text"),
],
)
def test_convert_numeric(value: bytes, expected: float | str) -> None:
result = _convert_numeric(value)
assert type(result) is type(expected)
assert result == expected


@pytest.mark.parametrize("value", [b"nan", b"NaN", b"-nan"])
def test_convert_numeric_nan(value: bytes) -> None:
result = _convert_numeric(value)
assert isinstance(result, float)
assert math.isnan(result)
Loading