From fb3af0c4805208aec569e4b4db90277a1e5b3db8 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Sun, 4 Feb 2024 16:22:53 +0000 Subject: [PATCH 1/3] keep all axes for all components --- activestorage/active.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/activestorage/active.py b/activestorage/active.py index d25b0618..7e491dc5 100644 --- a/activestorage/active.py +++ b/activestorage/active.py @@ -392,9 +392,9 @@ def _from_storage(self, stripped_indexer, drop_axes, out_shape, out_dtype, # cf-python needs the sample size for all reductions; # see the 'mtol' parameter of cf.Field.collapse.) # - # Note that in this case the reduced data must always - # have the same number of dimensions as the original - # array, i.e. 'drop_axes' is always considered False, + # Note that in all components must always have the + # same number of dimensions as the original array, + # i.e. 'drop_axes' is always considered False, # regardless of its setting. (Rationale: dask # reductions require the per-dask-chunk partial # reductions to retain these dimensions so that @@ -404,12 +404,13 @@ def _from_storage(self, stripped_indexer, drop_axes, out_shape, out_dtype, n = np.reshape(n, shape1) out = out.reshape(shape1) + n = np.sum(counts).reshape(shape1) if self._method == "mean": # For the average, the returned component is # "sum", not "mean" - out = {"sum": out, "n": sum(counts)} + out = {"sum": out, "n": n} else: - out = {self._method: out, "n": sum(counts)} + out = {self._method: out, "n": n} else: # Return the reduced data as a numpy array. For most # methods the data is already in this form. @@ -417,7 +418,7 @@ def _from_storage(self, stripped_indexer, drop_axes, out_shape, out_dtype, # For the average, it is actually the sum that has # been created, so we need to divide by the sample # size. - out = out / sum(counts) + out = out / np.sum(counts).reshape(shape1) return out From ac29a023ca3dc6ffadc810397149389f17f3da07 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Sun, 4 Feb 2024 16:27:40 +0000 Subject: [PATCH 2/3] keep all axes for all components --- activestorage/active.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/activestorage/active.py b/activestorage/active.py index 7e491dc5..55f58b60 100644 --- a/activestorage/active.py +++ b/activestorage/active.py @@ -399,9 +399,7 @@ def _from_storage(self, stripped_indexer, drop_axes, out_shape, out_dtype, # reductions require the per-dask-chunk partial # reductions to retain these dimensions so that # partial results can be concatenated correctly.) - n = np.prod(out_shape) shape1 = (1,) * len(out_shape) - n = np.reshape(n, shape1) out = out.reshape(shape1) n = np.sum(counts).reshape(shape1) From 714b83396e138a79758cbe500e25025dfafc900c Mon Sep 17 00:00:00 2001 From: David Hassell Date: Mon, 5 Feb 2024 08:16:02 +0000 Subject: [PATCH 3/3] move shape1 --- activestorage/active.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/activestorage/active.py b/activestorage/active.py index 55f58b60..fed673ca 100644 --- a/activestorage/active.py +++ b/activestorage/active.py @@ -385,7 +385,8 @@ def _from_storage(self, stripped_indexer, drop_axes, out_shape, out_dtype, if method is not None: # Apply the method (again) to aggregate the result out = method(out) - + shape1 = (1,) * len(out_shape) + if self._components: # Return a dictionary of components containing the # reduced data and the sample size ('n'). (Rationale: @@ -399,7 +400,6 @@ def _from_storage(self, stripped_indexer, drop_axes, out_shape, out_dtype, # reductions require the per-dask-chunk partial # reductions to retain these dimensions so that # partial results can be concatenated correctly.) - shape1 = (1,) * len(out_shape) out = out.reshape(shape1) n = np.sum(counts).reshape(shape1)