Missing and compression/filtering Issues

activestorage/active.py

@@ -119,7 +119,7 @@ def method(self):
     @method.setter
     def method(self, value):
         if value is not None and value not in self._methods:
-            raise ValueError(f"Bad 'method': {method}. Choose from min/max/mean/sum.")
+            raise ValueError(f"Bad 'method': {value}. Choose from min/max/mean/sum.")
 
         self._method = value
 
@@ -163,10 +163,13 @@ def _get_selection(self, *args):
         from zarr and friends and use simple dictionaries and tupes, then
         we can go to the storage layer with no zarr.
         """
-        if self.zds._compressor:
-            raise ValueError("No active support for compression as yet")
-        if self.zds._filters:
-            raise ValueError("No active support for filters as yet")
+        compressor = self.zds._compressor
+        filters = self.zds._filters
+
+        # FIXME: populate this from metadata, see issue #18
+        # interpretation: (_fillvalue, missing, min_valid_value, max_valid_value)
+        missing = (None, None, None, None)  # FIXME: Needs implementation 
+
 
         indexer = OrthogonalIndexer(*args, self.zds)
         out_shape = indexer.shape
@@ -177,19 +180,22 @@ def _get_selection(self, *args):
         # yes this next line is bordering on voodoo ... 
         fsref = self.zds.chunk_store._mutable_mapping.fs.references
 
-        return self._from_storage(stripped_indexer, drop_axes, out_shape, out_dtype, fsref)
-
-    def _from_storage(self, stripped_indexer, drop_axes, out_shape, out_dtype, fsref):
+        return self._from_storage(stripped_indexer, drop_axes, out_shape,
+                                  out_dtype, compressor, filters, missing, fsref)
 
+    def _from_storage(self, stripped_indexer, drop_axes, out_shape, out_dtype,
+                      compressor, filters, missing, fsref):
         method = self.method
         if method is not None:
             out = []
         else:
             out = np.empty(out_shape, dtype=out_dtype, order=self.zds._order)
 
         for chunk_coords, chunk_selection, out_selection in stripped_indexer:
-            self._process_chunk(fsref, chunk_coords,chunk_selection, out, out_selection,
-                                    drop_axes=drop_axes)
+            self._process_chunk(fsref, chunk_coords,chunk_selection,
+                                out, out_selection,
+                                compressor, filters, missing,
+                                drop_axes=drop_axes)
 
         if method is not None:
             # Apply the method (again) to aggregate the result
@@ -231,7 +237,8 @@ def _from_storage(self, stripped_indexer, drop_axes, out_shape, out_dtype, fsref
 
         return out
 
-    def _process_chunk(self, fsref, chunk_coords, chunk_selection, out, out_selection,
+    def _process_chunk(self, fsref, chunk_coords, chunk_selection, out,
+                       out_selection, compressor, filters, missing, 
                        drop_axes=None):
         """Obtain part or whole of a chunk.
 
@@ -242,8 +249,9 @@ def _process_chunk(self, fsref, chunk_coords, chunk_selection, out, out_selectio
         coord = '.'.join([str(c) for c in chunk_coords])
         key = f"{self.ncvar}/{coord}"
         rfile, offset, size = tuple(fsref[key])
-        tmp = decode_chunk(rfile, offset, size,
-                                 self.zds._dtype, self.zds._chunks, self.zds._order, chunk_selection, method=self.method)
+        tmp = decode_chunk(rfile, offset, size, compressor, filters, missing,
+                           self.zds._dtype, self.zds._chunks, self.zds._order,
+                           chunk_selection, method=self.method)
 
         if self.method is not None:
             out.append(tmp)

activestorage/storage.py

@@ -1,6 +1,7 @@
+from ast import Not
 from numcodecs.compat import ensure_ndarray
 
-def decode_chunk(rfile, offset, size, dtype, shape, order, chunk_selection, method=None):
+def decode_chunk(rfile, offset, size, compression, filters, missing, dtype, shape, order, chunk_selection, method=None):
         """ We do our own read of chunks and decoding etc 
 
         rfile - the actual file with the data 
@@ -19,6 +20,14 @@ def decode_chunk(rfile, offset, size, dtype, shape, order, chunk_selection, meth
 
         """
 
+        if compression is not None:
+            raise NotImplementedError
+        if filters is not None:
+            raise NotImplementedError
+        if missing is not (None, None, None, None):
+             #interpretation: (_fillvalue, missing, min_valid_value, max_valid_value)
+            raise NotImplementedError
+
         #fIXME: for the moment, open the file every time ... we might want to do that, or not
         with open(rfile,'rb') as open_file:
             # get the data