From 6cb3c8d086a314dfdc422ed9789a51b015088610 Mon Sep 17 00:00:00 2001 From: Reema Gupta Date: Mon, 20 Feb 2023 18:58:42 +0100 Subject: [PATCH 1/5] temporarily fixed 0 LT size and NoneType block errors --- neo/rawio/monkeylogicrawio.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/neo/rawio/monkeylogicrawio.py b/neo/rawio/monkeylogicrawio.py index 4d79eae6d..53ff34284 100644 --- a/neo/rawio/monkeylogicrawio.py +++ b/neo/rawio/monkeylogicrawio.py @@ -41,6 +41,9 @@ def generate_block(f): # print(var_name) LT = f.read(8) + # print(len(LT)) + if len(LT) == 0: + return None LT = struct.unpack('Q', LT)[0] # print(f'LT: {LT}') var_type = f.read(LT) @@ -132,9 +135,12 @@ def read_data(self, f, recursive=False): for field in range(n_fields * np.prod(self.var_size)): bl = MLBlock.generate_block(f) - if recursive: - self[bl.var_name] = bl - bl.read_data(f, recursive=recursive) + if bl is None: + pass + else: + if recursive: + self[bl.var_name] = bl + bl.read_data(f, recursive=recursive) elif self.var_type == 'cell': # cells are always 2D @@ -144,13 +150,17 @@ def read_data(self, f, recursive=False): bl = MLBlock.generate_block(f) if recursive: data[field] = bl + + if bl is None: + pass + else: + bl.read_data(f, recursive=recursive) - bl.read_data(f, recursive=recursive) data = data.reshape(self.var_size) self.data = data - else: - raise ValueError(f'unknown variable type {self.var_type}') + # else: + # raise ValueError(f'unknown variable type {self.var_type}') self.flatten() From e46bbe397b1e75e2e46f19da425d199e9ffafc11 Mon Sep 17 00:00:00 2001 From: Reema Gupta Date: Mon, 27 Feb 2023 09:26:14 +0100 Subject: [PATCH 2/5] Fixed some errors handling variables --- neo/rawio/monkeylogicrawio.py | 55 +++++++++++++++++++++++++++++++---- 1 file changed, 49 insertions(+), 6 deletions(-) diff --git a/neo/rawio/monkeylogicrawio.py b/neo/rawio/monkeylogicrawio.py index 53ff34284..406a9d39d 100644 --- a/neo/rawio/monkeylogicrawio.py +++ b/neo/rawio/monkeylogicrawio.py @@ -22,7 +22,8 @@ class MLBlock(dict): 'integers': (8, 'Q'), 'uint8': (1, 'B'), 'single': (4, 'f'), - 'double': (8, 'd')} + 'double': (8, 'd'), + 'uint64': (8, 'L')} @staticmethod def generate_block(f): @@ -79,6 +80,11 @@ def __repr__(self): if self.data is None: shape = 0 dt = '' + + elif not hasattr(self.data, '__len__'): + shape = 1 + dt = f' dtype: {self.var_type}' + else: shape = getattr(self.data, 'shape', len(self.data)) dt = f' dtype: {self.var_type}' @@ -159,8 +165,11 @@ def read_data(self, f, recursive=False): data = data.reshape(self.var_size) self.data = data - # else: - # raise ValueError(f'unknown variable type {self.var_type}') + elif self.var_type == 'function_handle': + pass + + else: + raise ValueError(f'unknown variable type {self.var_type}') self.flatten() @@ -226,6 +235,9 @@ def _parse_header(self): exclude_signals = ['SampleInterval'] + print(self.ml_blocks.keys()) + print(self.ml_blocks.values()) + # rawio configuration signal_streams = [] signal_channels = [] @@ -240,14 +252,40 @@ def _parse_header(self): def _register_signal(sig_block, name): nonlocal stream_id nonlocal chan_id - if not isinstance(sig_data, dict) and any(sig_data.shape): + if not isinstance(sig_data, dict) and any(np.shape(sig_data)): signal_streams.append((name, stream_id)) sr = 1 # TODO: Where to get the sampling rate info? + # ML/Trial1/AnalogData/SampleInterval + # ML/MLConfig/HighFrequencyDAQ/SampleRate + # ML/MLConfig/VoiceRecording/SampleRate + # ML/MLConfig/AISampleRate + # ML/TrialRecord/LastTrialAnalogData/SampleInterval + dtype = type(sig_data) units = '' # TODO: Where to find the unit info? + # Degree of visual angle is default coordinate system used by ML, see here: + # https://monkeylogic.nimh.nih.gov/docs_CoordinateConversion.html + + # /ML/MLConfig/DiagonalSize + # /ML/MLConfig/ViewingDistance + + # ML/MLConfig/Screen has details about screen, specifically: + # /ML/MLConfig/Screen/Xsize + # /ML/MLConfig/Screen/Ysize + # /ML/MLConfig/Screen/PixelsPerDegree + # /ML/MLConfig/Screen/RefreshRate + # /ML/MLConfig/Screen/FrameLength + # /ML/MLConfig/Screen/VBlankLength + gain = 1 # TODO: Where to find the gain info? + + # ML/MLConfig/Webcam/1/Property/Gain + offset = 0 # TODO: Can signals have an offset in ML? + + # /ML/MLConfig/EyeTransform/2/offset if it exists + stream_id = 0 # all analog data belong to same stream if sig_block.shape[1] == 1: @@ -264,9 +302,14 @@ def _register_signal(sig_block, name): for sig_name, sig_data in ana_block.items(): if sig_name in exclude_signals: continue + + print(sig_name) + print(sig_data) + # print(sig_sub_name) + # print(sig_sub_data) # 1st level signals ('Trial1'/'AnalogData'/') - if not isinstance(sig_data, dict) and any(sig_data.shape): + if not isinstance(sig_data, dict) and any(np.shape(sig_data)): _register_signal(sig_data, name=sig_name) # 2nd level signals ('Trial1'/'AnalogData'//') @@ -337,7 +380,7 @@ def _filter_keys(full_dict, ignore_keys, simplify=True): seg_ann = self.raw_annotations['blocks'][0]['segments'][trial_id-1] seg_ann.update(_filter_keys(ml_trial, ignore_annotations)) - event_ann = seg_ann['events'][0] # 0 is event + # event_ann = seg_ann['events'][0] # 0 is event # epoch_ann = seg_ann['events'][1] # 1 is epoch def _segment_t_start(self, block_index, seg_index): From 2eb36871f5d3ac3b55501b352b72dd263ee5d2d7 Mon Sep 17 00:00:00 2001 From: Julia Sprenger Date: Tue, 7 Mar 2023 12:40:03 +0100 Subject: [PATCH 3/5] normalize annotation types (ndarray are not accepted as annotation values) --- neo/rawio/monkeylogicrawio.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/neo/rawio/monkeylogicrawio.py b/neo/rawio/monkeylogicrawio.py index 406a9d39d..42002627a 100644 --- a/neo/rawio/monkeylogicrawio.py +++ b/neo/rawio/monkeylogicrawio.py @@ -11,6 +11,7 @@ import numpy as np import struct +from typing import Union from .baserawio import (BaseRawIO, _signal_channel_dtype, _signal_stream_dtype, _spike_channel_dtype, _event_channel_dtype) @@ -370,6 +371,29 @@ def _filter_keys(full_dict, ignore_keys, simplify=True): ml_ann = {k: v for k, v in self.ml_blocks.items() if k in ['MLConfig', 'TrialRecord']} ml_ann = _filter_keys(ml_ann, ignore_annotations) + # normalize annotation values, convert arrays to lists + + def recursively_replace_arrays(container: Union[dict, list]) -> None: + """ + Replace numpy arrays in nested dictionary and list structures by lists + """" + + if isinstance(container, dict): + iterator = container.items() + elif isinstance(container, list): + iterator = enumerate(container) + + for k, v in iterator: + if isinstance(v, dict): + recursively_replace_arrays(v) + elif isinstance(v, np.ndarray): + container[k] = list(v) + recursively_replace_arrays(container[k]) + elif isinstance(v, list): + recursively_replace_arrays(container[k]) + + recursively_replace_arrays(ml_ann) + bl_ann = self.raw_annotations['blocks'][0] bl_ann.update(ml_ann) @@ -377,6 +401,8 @@ def _filter_keys(full_dict, ignore_keys, simplify=True): ml_trial = self.ml_blocks[f'Trial{trial_id}'] assert ml_trial['Trial'] == trial_id + recursively_replace_arrays(ml_trial) + seg_ann = self.raw_annotations['blocks'][0]['segments'][trial_id-1] seg_ann.update(_filter_keys(ml_trial, ignore_annotations)) From b0b3e2fd3d48b28cdc373bba9c52fa5b2948eed9 Mon Sep 17 00:00:00 2001 From: Julia Sprenger Date: Thu, 9 Mar 2023 15:58:02 +0100 Subject: [PATCH 4/5] fix syntax --- neo/rawio/monkeylogicrawio.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neo/rawio/monkeylogicrawio.py b/neo/rawio/monkeylogicrawio.py index 42002627a..5a80abc3f 100644 --- a/neo/rawio/monkeylogicrawio.py +++ b/neo/rawio/monkeylogicrawio.py @@ -376,7 +376,7 @@ def _filter_keys(full_dict, ignore_keys, simplify=True): def recursively_replace_arrays(container: Union[dict, list]) -> None: """ Replace numpy arrays in nested dictionary and list structures by lists - """" + """ if isinstance(container, dict): iterator = container.items() From 34ffa92e3c9d396304e43ddfa5f6d25364ed19ae Mon Sep 17 00:00:00 2001 From: Reema Gupta Date: Mon, 13 Mar 2023 11:22:49 +0100 Subject: [PATCH 5/5] Using a tuple to store data shape --- neo/rawio/monkeylogicrawio.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neo/rawio/monkeylogicrawio.py b/neo/rawio/monkeylogicrawio.py index 406a9d39d..06ecea93c 100644 --- a/neo/rawio/monkeylogicrawio.py +++ b/neo/rawio/monkeylogicrawio.py @@ -82,7 +82,7 @@ def __repr__(self): dt = '' elif not hasattr(self.data, '__len__'): - shape = 1 + shape = (1, ) dt = f' dtype: {self.var_type}' else: