Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,22 @@
# `open-ephys-python-tools` Changelog

## Unreleased

- Support the NWB files written by the AqNWB-based NWB Format plugin (NWB 2.11); files written by earlier
versions of the plugin can still be read
- Sample numbers are read from `sync/sample_numbers`, and the sample rate from `starting_time.rate` or the
timestamps (instead of `timestamps.interval`, which NWB fixes to 1)
- Continuous data and spikes are scaled with `data.conversion * channel_conversion`, as specified by NWB
(files of earlier plugin versions store the factor in both, so only `channel_conversion` is applied for them)
- TTL events are read from the `EventsTable`s in `/events`, which are named as the Binary event folders
(`<processor>-<id>.<stream>.TTL`)
- `NwbRecording.load_messages` is implemented (new and previous NWB layout)
- NWB files are opened by the names found in the Record Node directory, `experiment<N>.nwb` and
`experiment<N>_recording<M>.nwb` (written by the plugin if `experiment<N>.nwb` already exists), and ordered by
experiment and recording number (previously, the file names were derived from the position in the alphabetically
sorted list, which failed for missing experiment numbers and for 10 or more experiments)
- Added a v1.1.0 NWB test file and tests

## 1.0.0

- Dropped support for Python < 3.9
Expand Down
224 changes: 167 additions & 57 deletions src/open_ephys/analysis/formats/NwbRecording.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@

import glob
import os
import re
import h5py as h5
import numpy as np
import pandas as pd
Expand All @@ -40,6 +41,44 @@
)


def _is_legacy(group: h5.Group) -> bool:
"""Files written by the NWB plugin before v2 (AqNWB) store the sample numbers in a ``sync`` dataset"""
return isinstance(group.get("sync"), h5.Dataset)


def _sample_numbers(group: h5.Group):
"""Sample numbers of a TimeSeries: ``sync/sample_numbers`` (NWB plugin >= 2) or the legacy ``sync`` dataset"""
sync = group.get("sync")
if isinstance(sync, h5.Dataset):
return sync
if isinstance(sync, h5.Group) and "sample_numbers" in sync:
return sync["sample_numbers"]
return None


def _conversion_to_microvolts(group: h5.Group) -> np.ndarray:
"""Per-channel factors to convert the stored values of an ElectricalSeries/SpikeEventSeries to microvolts.

Files written by the NWB plugin before v2 store the factor both in ``data.conversion`` and in
``channel_conversion`` (i.e., applying both would scale twice), so only ``channel_conversion`` is used.
Otherwise, values in volts are ``data * data.conversion * channel_conversion`` (NWB specification).
"""
channel_conversion = group["channel_conversion"][()] if "channel_conversion" in group else np.array([1.0])
conversion = 1.0 if _is_legacy(group) else float(group["data"].attrs.get("conversion", 1.0))
return channel_conversion * conversion * 1e6


def _sample_rate(group: h5.Group) -> float:
"""Sample rate from ``starting_time.rate``, or estimated from the timestamps"""
if "starting_time" in group:
return float(group["starting_time"].attrs["rate"])
timestamps = group["timestamps"][:1001]
if len(timestamps) > 1:
return float(np.around(1 / np.median(np.diff(timestamps)), 1))
# very short legacy series: earlier versions of the plugin stored 1/sample_rate in timestamps.interval
return float(np.around(1 / group["timestamps"].attrs["interval"], 1))


class NwbSpikes(Spikes):
def __init__(self, nwb: h5.File, dataset: str):
self.metadata = SpikeMetadata(
Expand All @@ -49,11 +88,14 @@ def __init__(self, nwb: h5.File, dataset: str):
sample_rate=None, # FIXME:
)

self.timestamps = nwb["acquisition"][dataset]["timestamps"][()]
self.sample_numbers = nwb["acquisition"][dataset]["sync"][()]
self.waveforms = nwb["acquisition"][dataset]["data"][()].astype("float64")
group = nwb["acquisition"][dataset]
self.timestamps = group["timestamps"][()]
sample_numbers = _sample_numbers(group)
self.sample_numbers = sample_numbers[()] if sample_numbers is not None else None
self.waveforms = group["data"][()].astype("float64")

self.waveforms *= nwb["acquisition"][dataset]["channel_conversion"][0] * 1e6
# all channels of a spike waveform use the same factor
self.waveforms *= _conversion_to_microvolts(group)[0]


class NwbContinuous(Continuous):
Expand All @@ -64,20 +106,24 @@ def __init__(self, nwb: h5.File, dataset: str):
source_node_id = int(source_node[-3:])
source_node_name = source_node[:-4]

group = nwb["acquisition"][dataset]
num_channels = group["data"].shape[1]
bit_volts = _conversion_to_microvolts(group)
if len(bit_volts) == 1:
bit_volts = np.repeat(bit_volts, num_channels)

self.metadata = ContinuousMetadata(
source_node_id=source_node_id,
source_node_name=source_node_name,
stream_name=stream_name,
sample_rate=np.around(
1 / nwb["acquisition"][dataset]["timestamps"].attrs["interval"], 1
),
num_channels=nwb["acquisition"][dataset]["data"].shape[1],
bit_volts=list(nwb["acquisition"][dataset]["channel_conversion"][()] * 1e6),
sample_rate=_sample_rate(group),
num_channels=num_channels,
bit_volts=list(bit_volts),
channel_names=None, # TODO: add this
)
self.samples = nwb["acquisition"][dataset]["data"]
self.sample_numbers = nwb["acquisition"][dataset]["sync"]
self.timestamps = nwb["acquisition"][dataset]["timestamps"]
self.samples = group["data"]
self.sample_numbers = _sample_numbers(group)
self.timestamps = group["timestamps"]

self.global_timestamps = None

Expand Down Expand Up @@ -136,16 +182,19 @@ def get_samples(

class NwbRecording(Recording):

def __init__(self, directory: str, experiment_index=0, recording_index=0):
def __init__(
self, directory: str, experiment_index=0, recording_index=0, file_name=None
):
"""
file_name: name of the NWB file in the directory (default: experiment<experiment_index + 1>.nwb)
"""

Recording.__init__(self, directory, experiment_index, recording_index)
self._format = RecordingFormat.nwb
self.nwb = h5.File(
os.path.join(
self.directory, "experiment" + str(self.experiment_index + 1) + ".nwb"
),
"r",
)
if file_name is None:
file_name = "experiment" + str(self.experiment_index + 1) + ".nwb"
self.file_name = file_name
self.nwb = h5.File(os.path.join(self.directory, file_name), "r")

def __delete__(self):

Expand Down Expand Up @@ -193,49 +242,93 @@ def load_spikes(self):

def load_events(self):

datasets = list(self.nwb["acquisition"].keys())

events = []
processor_ids = []
stream_indexes = {}

def add_events(processor_id, stream_name, lines, states, timestamps, sample_numbers):
if len(lines) == 0:
return # empty frames would turn the integer columns into floats in pd.concat
# stream_index counts the event streams of each processor
key = (processor_id, stream_name)
if key not in stream_indexes:
stream_indexes[key] = sum(1 for p, _ in stream_indexes if p == processor_id)
events.append(
pd.DataFrame(
data={
"line": lines,
"timestamp": timestamps,
"sample_number": sample_numbers,
# float, as in the Binary format and earlier versions of this reader
"processor_id": np.full(len(lines), processor_id, dtype="float64"),
"stream_index": np.full(len(lines), stream_indexes[key], dtype="float64"),
"stream_name": [stream_name] * len(lines),
"state": states,
}
)
)

for dataset in datasets:
# NWB plugin >= 2: one EventsTable per TTL event channel in /events, named as the event folders of the
# Binary format: "<processor>-<id>.<stream>.TTL" (further channels of the processor: "TTL_1", ...)
for name, table in (self.nwb["events"].items() if "events" in self.nwb else []):
if not {"line", "state", "sample_number", "timestamp"} <= set(table.keys()):
continue
parts = name.split(".")
if len(parts) < 3 or not (parts[-1] == "TTL" or parts[-1].startswith("TTL_")):
continue
processor_id = int(parts[0].rsplit("-", 1)[-1])
add_events(
processor_id,
".".join(parts[1:-1]),
table["line"][()].astype("int16") + 1, # 1-based int16, as in the Binary format
table["state"][()].astype("int"),
table["timestamp"][()],
table["sample_number"][()],
)

# NWB plugin < 2: TTL events as "<source>-<id>.<stream>.TTL" TimeSeries with data = +/-(line + 1)
for dataset in self.nwb["acquisition"].keys():
if dataset[-4:] == ".TTL":

processor_id = int(dataset.split(".")[0].split("-")[-1])
stream_name = dataset.split(".")[1]

if processor_id not in processor_ids:
processor_ids.append(processor_id)
stream_id = 0
else:
stream_id += 1

ds = self.nwb["acquisition"][dataset]
channel_states = ds["data"]
sample_numbers = ds["sync"]
timestamps = ds["timestamps"]

events.append(
pd.DataFrame(
data={
"line": np.abs(channel_states),
"timestamp": timestamps,
"sample_number": sample_numbers,
"processor_id": [processor_id] * len(channel_states),
"stream_index": [stream_id] * len(channel_states),
"stream_name": [stream_name] * len(channel_states),
"state": (np.sign(channel_states) + 1 / 2).astype("int"),
}
)
channel_states = ds["data"][()]
add_events(
int(dataset.split(".")[0].split("-")[-1]),
dataset.split(".")[1],
np.abs(channel_states),
(channel_states > 0).astype("int"),
ds["timestamps"][()],
ds["sync"][()],
)
self._events = pd.concat(events).sort_values(
by=["sample_number", "stream_index"], ignore_index=True
)

if len(events) > 0:
self._events = pd.concat(events).sort_values(
by=["sample_number", "stream_index"], ignore_index=True
)
else:
self._events = None

def load_messages(self):
raise NotImplementedError(
"Loading messages from NWB2 format is noy yet implemented."

# NWB plugin >= 2: EventsTable /events/messages; NWB plugin < 2: AnnotationSeries /acquisition/messages
if "events" in self.nwb and "messages" in self.nwb["events"]:
table = self.nwb["events"]["messages"]
sample_numbers = table["sample_number"][()]
timestamps = table["timestamp"][()]
text = table["annotation"][()]
elif "messages" in self.nwb["acquisition"] and isinstance(self.nwb["acquisition"]["messages"].get("sync"), h5.Dataset):
series = self.nwb["acquisition"]["messages"]
sample_numbers = series["sync"][()]
timestamps = series["timestamps"][()]
text = series["data"][()]
else:
self._messages = None
return

self._messages = pd.DataFrame(
data={
"sample_number": sample_numbers,
"timestamp": timestamps,
"message": [t.decode("utf-8") if isinstance(t, bytes) else str(t) for t in text],
}
)

def __str__(self):
Expand Down Expand Up @@ -273,10 +366,20 @@ def detect_recordings(directory, mmap_timestamps=True):

recordings = []

nwb_files = glob.glob(os.path.join(directory, "experiment*.nwb"))
# The plugin writes one file per experiment, experiment<N>.nwb. If the directory already contains that file
# when a recording starts (e.g., after the data directory was changed between recordings and back), the
# file is named after the experiment and its first recording instead: experiment<N>_recording<M>.nwb
name_pattern = re.compile(r"experiment(\d+)(?:_recording(\d+))?\.nwb$")
nwb_files = []
for file in glob.glob(os.path.join(directory, "experiment*.nwb")):
match = name_pattern.match(os.path.basename(file))
if match:
nwb_files.append(
(int(match.group(1)), int(match.group(2) or 1), file)
)
nwb_files.sort()

for experiment_index, file in enumerate(nwb_files):
for experiment_number, recording_number, file in nwb_files:

try:
f = h5.File(file, "r")
Expand All @@ -287,7 +390,14 @@ def detect_recordings(directory, mmap_timestamps=True):
+ "\nis likely still in use. Try closing the GUI and re-loading the session object."
)
else:
recordings.append(NwbRecording(directory, experiment_index, 0))
recordings.append(
NwbRecording(
directory,
experiment_number - 1,
recording_number - 1,
os.path.basename(file),
)
)
f.close()

return recordings
Binary file not shown.
Loading