add extra kwarg to request_data

Author: valeriupredoi

activestorage/active.py

@@ -243,9 +243,6 @@ def __load_nc_file(self):
             nc = load_from_s3(self.uri, self.storage_options)
         elif self.storage_type == "https":
             nc = load_from_https(self.uri)
-        # testing only
-        elif self.storage_type == "https-Reductionist":
-            nc = load_from_https(self.uri)
         self.filename = self.uri
         self.ds = nc[ncvar]
 
@@ -522,7 +519,7 @@ def _process_chunk(self, session, ds, chunks, chunk_coords, chunk_selection, cou
                             chunk_selection, method=self.method
             )
 
-        elif self.storage_type == "s3" and self._version==2:
+        elif self.storage_type == "s3" and self._version == 2:
             # S3: pass in pre-configured storage options (credentials)
             # print("S3 rfile is:", self.filename)
             parsed_url = urllib.parse.urlparse(self.filename)
@@ -570,13 +567,14 @@ def _process_chunk(self, session, ds, chunks, chunk_coords, chunk_selection, cou
         # this is for testing ONLY until Reductionist is able to handle https
         # located files; after that, we can pipe any regular https file through
         # to Reductionist, provided the https server is "closer" to Reductionist
-        elif self.storage_type == "https-Reductionist" and self._version==2:
+        elif self.storage_type == "https" and self._version == 2:
             # build a simple session
             session = requests.Session()
             session.auth = (None, None)
             session.verify = False
             bucket = "https"
 
+            # note the extra "storage_type" kwarg
             tmp, count = reductionist.reduce_chunk(session,
                                                    "https://192.171.169.113:8080",
                                                    self.filename,
@@ -586,7 +584,8 @@ def _process_chunk(self, session, ds, chunks, chunk_coords, chunk_selection, cou
                                                    chunks,
                                                    ds._order,
                                                    chunk_selection,
-                                                   operation=self._method)
+                                                   operation=self._method,
+                                                   storage_type="https")
         elif self.storage_type=='ActivePosix' and self.version==2:
             # This is where the DDN Fuse and Infinia wrappers go
             raise NotImplementedError

activestorage/reductionist.py

@@ -27,7 +27,7 @@ def get_session(username: str, password: str, cacert: typing.Optional[str]) -> r
 
 def reduce_chunk(session, server, source, bucket, object,
                  offset, size, compression, filters, missing, dtype, shape,
-                 order, chunk_selection, operation):
+                 order, chunk_selection, operation, storage_type=None):
     """Perform a reduction on a chunk using Reductionist.
 
     :param server: Reductionist server URL
@@ -50,12 +50,14 @@ def reduce_chunk(session, server, source, bucket, object,
                             this defines the part of the chunk which is to be
                             obtained or operated upon.
     :param operation: name of operation to perform
+    :param storage_type: optional testing flag to allow HTTPS reduction
     :returns: the reduced data as a numpy array or scalar
     :raises ReductionistError: if the request to Reductionist fails
     """
 
     request_data = build_request_data(source, bucket, object, offset, size, compression,
-                                      filters, missing, dtype, shape, order, chunk_selection)
+                                      filters, missing, dtype, shape, order, chunk_selection,
+                                      storage_type=storage_type)
     if DEBUG:
         print(f"Reductionist request data dictionary: {request_data}")
     api_operation = "sum" if operation == "mean" else operation or "select"
@@ -135,7 +137,7 @@ def encode_missing(missing):
 
 def build_request_data(source: str, bucket: str, object: str, offset: int,
                        size: int, compression, filters, missing, dtype, shape,
-                       order, selection) -> dict:
+                       order, selection, storage_type=None) -> dict:
     """Build request data for Reductionist API."""
     request_data = {
         'source': source,
@@ -146,6 +148,7 @@ def build_request_data(source: str, bucket: str, object: str, offset: int,
         'offset': int(offset),
         'size': int(size),
         'order': order,
+        'storage_type': storage_type,
     }
     if shape:
         request_data["shape"] = shape