From dd48e12c9e6ef6aeef51ce8d122843d7a5d0797e Mon Sep 17 00:00:00 2001 From: Bryan Lawrence Date: Thu, 13 Oct 2022 10:47:43 +0100 Subject: [PATCH 1/7] moved all the necessary arguments down to _decode_chunk, none yet implemented --- activestorage/active.py | 18 +++++++++--------- activestorage/storage.py | 10 +++++++++- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/activestorage/active.py b/activestorage/active.py index afa0da0a..7f45c04a 100644 --- a/activestorage/active.py +++ b/activestorage/active.py @@ -163,10 +163,10 @@ def _get_selection(self, *args): from zarr and friends and use simple dictionaries and tupes, then we can go to the storage layer with no zarr. """ - if self.zds._compressor: - raise ValueError("No active support for compression as yet") - if self.zds._filters: - raise ValueError("No active support for filters as yet") + compressor = self.zds._compressor + filters = self.zds._filters + missing = None #FIXME: Needs implementation + indexer = OrthogonalIndexer(*args, self.zds) out_shape = indexer.shape @@ -177,9 +177,9 @@ def _get_selection(self, *args): # yes this next line is bordering on voodoo ... fsref = self.zds.chunk_store._mutable_mapping.fs.references - return self._from_storage(stripped_indexer, drop_axes, out_shape, out_dtype, fsref) + return self._from_storage(stripped_indexer, drop_axes, out_shape, out_dtype, compressor, filters, missing, fsref) - def _from_storage(self, stripped_indexer, drop_axes, out_shape, out_dtype, fsref): + def _from_storage(self, stripped_indexer, drop_axes, out_shape, out_dtype, compressor, filters, missing, fsref): method = self.method if method is not None: @@ -188,7 +188,7 @@ def _from_storage(self, stripped_indexer, drop_axes, out_shape, out_dtype, fsref out = np.empty(out_shape, dtype=out_dtype, order=self.zds._order) for chunk_coords, chunk_selection, out_selection in stripped_indexer: - self._process_chunk(fsref, chunk_coords,chunk_selection, out, out_selection, + self._process_chunk(fsref, chunk_coords,chunk_selection, out, out_selection, compressor, filters, missing, drop_axes=drop_axes) if method is not None: @@ -231,7 +231,7 @@ def _from_storage(self, stripped_indexer, drop_axes, out_shape, out_dtype, fsref return out - def _process_chunk(self, fsref, chunk_coords, chunk_selection, out, out_selection, + def _process_chunk(self, fsref, chunk_coords, chunk_selection, out, out_selection, compressor, filters, missing, drop_axes=None): """Obtain part or whole of a chunk. @@ -242,7 +242,7 @@ def _process_chunk(self, fsref, chunk_coords, chunk_selection, out, out_selectio coord = '.'.join([str(c) for c in chunk_coords]) key = f"{self.ncvar}/{coord}" rfile, offset, size = tuple(fsref[key]) - tmp = decode_chunk(rfile, offset, size, + tmp = decode_chunk(rfile, offset, size, compressor, filters, missing, self.zds._dtype, self.zds._chunks, self.zds._order, chunk_selection, method=self.method) if self.method is not None: diff --git a/activestorage/storage.py b/activestorage/storage.py index 757d079c..3d548bc1 100644 --- a/activestorage/storage.py +++ b/activestorage/storage.py @@ -1,6 +1,7 @@ +from ast import Not from numcodecs.compat import ensure_ndarray -def decode_chunk(rfile, offset, size, dtype, shape, order, chunk_selection, method=None): +def decode_chunk(rfile, offset, size, compression, filters, missing, dtype, shape, order, chunk_selection, method=None): """ We do our own read of chunks and decoding etc rfile - the actual file with the data @@ -19,6 +20,13 @@ def decode_chunk(rfile, offset, size, dtype, shape, order, chunk_selection, meth """ + if compression is not None: + raise NotImplementedError + if filters is not None: + raise NotImplementedError + if missing is not None: + raise NotImplementedError + #fIXME: for the moment, open the file every time ... we might want to do that, or not with open(rfile,'rb') as open_file: # get the data From 3b44ad76519d82290fb782675e4b35efb6796eb1 Mon Sep 17 00:00:00 2001 From: Bryan Lawrence Date: Thu, 13 Oct 2022 10:49:42 +0100 Subject: [PATCH 2/7] fixed minor error in method error handling --- activestorage/active.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/activestorage/active.py b/activestorage/active.py index 7f45c04a..224595ce 100644 --- a/activestorage/active.py +++ b/activestorage/active.py @@ -119,7 +119,7 @@ def method(self): @method.setter def method(self, value): if value is not None and value not in self._methods: - raise ValueError(f"Bad 'method': {method}. Choose from min/max/mean/sum.") + raise ValueError(f"Bad 'method': {value}. Choose from min/max/mean/sum.") self._method = value From 967a94c640d386b8c827dde8ce68b4005e1dde6d Mon Sep 17 00:00:00 2001 From: Bryan Lawrence Date: Thu, 13 Oct 2022 10:54:53 +0100 Subject: [PATCH 3/7] added more detail to the missing interface --- activestorage/active.py | 7 +++++-- activestorage/storage.py | 3 ++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/activestorage/active.py b/activestorage/active.py index 224595ce..858a842f 100644 --- a/activestorage/active.py +++ b/activestorage/active.py @@ -164,8 +164,11 @@ def _get_selection(self, *args): we can go to the storage layer with no zarr. """ compressor = self.zds._compressor - filters = self.zds._filters - missing = None #FIXME: Needs implementation + filters = self.zds._filters + + #FIXME: populate this from metadata, see issue #18 + #interpretation: (_fillvalue, missing, min_valid_value, max_valid_value) + missing = (None,None,None,None) #FIXME: Needs implementation indexer = OrthogonalIndexer(*args, self.zds) diff --git a/activestorage/storage.py b/activestorage/storage.py index 3d548bc1..f3e7d985 100644 --- a/activestorage/storage.py +++ b/activestorage/storage.py @@ -24,7 +24,8 @@ def decode_chunk(rfile, offset, size, compression, filters, missing, dtype, shap raise NotImplementedError if filters is not None: raise NotImplementedError - if missing is not None: + if missing is not (None, None, None, None): + #interpretation: (_fillvalue, missing, min_valid_value, max_valid_value) raise NotImplementedError #fIXME: for the moment, open the file every time ... we might want to do that, or not From e36b8b787415b4d89f6e591afc5f453654a04ad1 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Thu, 13 Oct 2022 14:12:11 +0100 Subject: [PATCH 4/7] fixing formatting --- activestorage/active.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/activestorage/active.py b/activestorage/active.py index 858a842f..d7ec054a 100644 --- a/activestorage/active.py +++ b/activestorage/active.py @@ -166,9 +166,9 @@ def _get_selection(self, *args): compressor = self.zds._compressor filters = self.zds._filters - #FIXME: populate this from metadata, see issue #18 - #interpretation: (_fillvalue, missing, min_valid_value, max_valid_value) - missing = (None,None,None,None) #FIXME: Needs implementation + # FIXME: populate this from metadata, see issue #18 + # interpretation: (_fillvalue, missing, min_valid_value, max_valid_value) + missing = (None, None, None, None) # FIXME: Needs implementation indexer = OrthogonalIndexer(*args, self.zds) From d0a42f2c1a323ef9e3df17f6279a9d1fa57c0f97 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Thu, 13 Oct 2022 14:12:43 +0100 Subject: [PATCH 5/7] fixing formatting --- activestorage/active.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/activestorage/active.py b/activestorage/active.py index d7ec054a..ee2c48cc 100644 --- a/activestorage/active.py +++ b/activestorage/active.py @@ -191,8 +191,9 @@ def _from_storage(self, stripped_indexer, drop_axes, out_shape, out_dtype, compr out = np.empty(out_shape, dtype=out_dtype, order=self.zds._order) for chunk_coords, chunk_selection, out_selection in stripped_indexer: - self._process_chunk(fsref, chunk_coords,chunk_selection, out, out_selection, compressor, filters, missing, - drop_axes=drop_axes) + self._process_chunk(fsref, chunk_coords,chunk_selection, out, out_selection, + compressor, filters, missing, + drop_axes=drop_axes) if method is not None: # Apply the method (again) to aggregate the result From d1e794584b4e7cb37d08481d3a97b4655d967b87 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Thu, 13 Oct 2022 14:14:28 +0100 Subject: [PATCH 6/7] fix wonky GH formatted via browser --- activestorage/active.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/activestorage/active.py b/activestorage/active.py index ee2c48cc..52871e65 100644 --- a/activestorage/active.py +++ b/activestorage/active.py @@ -191,9 +191,10 @@ def _from_storage(self, stripped_indexer, drop_axes, out_shape, out_dtype, compr out = np.empty(out_shape, dtype=out_dtype, order=self.zds._order) for chunk_coords, chunk_selection, out_selection in stripped_indexer: - self._process_chunk(fsref, chunk_coords,chunk_selection, out, out_selection, - compressor, filters, missing, - drop_axes=drop_axes) + self._process_chunk(fsref, chunk_coords,chunk_selection, + out, out_selection, + compressor, filters, missing, + drop_axes=drop_axes) if method is not None: # Apply the method (again) to aggregate the result From ab372a4271f93a2b0bce72e0860e5bbddf5afb45 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Thu, 13 Oct 2022 14:16:59 +0100 Subject: [PATCH 7/7] few lines too long fixed --- activestorage/active.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/activestorage/active.py b/activestorage/active.py index 52871e65..cecd68ff 100644 --- a/activestorage/active.py +++ b/activestorage/active.py @@ -180,10 +180,11 @@ def _get_selection(self, *args): # yes this next line is bordering on voodoo ... fsref = self.zds.chunk_store._mutable_mapping.fs.references - return self._from_storage(stripped_indexer, drop_axes, out_shape, out_dtype, compressor, filters, missing, fsref) - - def _from_storage(self, stripped_indexer, drop_axes, out_shape, out_dtype, compressor, filters, missing, fsref): + return self._from_storage(stripped_indexer, drop_axes, out_shape, + out_dtype, compressor, filters, missing, fsref) + def _from_storage(self, stripped_indexer, drop_axes, out_shape, out_dtype, + compressor, filters, missing, fsref): method = self.method if method is not None: out = [] @@ -236,7 +237,8 @@ def _from_storage(self, stripped_indexer, drop_axes, out_shape, out_dtype, compr return out - def _process_chunk(self, fsref, chunk_coords, chunk_selection, out, out_selection, compressor, filters, missing, + def _process_chunk(self, fsref, chunk_coords, chunk_selection, out, + out_selection, compressor, filters, missing, drop_axes=None): """Obtain part or whole of a chunk. @@ -248,7 +250,8 @@ def _process_chunk(self, fsref, chunk_coords, chunk_selection, out, out_selectio key = f"{self.ncvar}/{coord}" rfile, offset, size = tuple(fsref[key]) tmp = decode_chunk(rfile, offset, size, compressor, filters, missing, - self.zds._dtype, self.zds._chunks, self.zds._order, chunk_selection, method=self.method) + self.zds._dtype, self.zds._chunks, self.zds._order, + chunk_selection, method=self.method) if self.method is not None: out.append(tmp)