test runner

Author: Bryan Lawrence

bnl/bnl_timeseries.py

@@ -0,0 +1,68 @@
+# Simple code to get hemispheric mean of the lowest level of air temperature from 
+# float UM_m01s30i204_vn1106(time, air_pressure_3, latitude_1, longitude_1) ;
+#		UM_m01s30i204_vn1106:long_name = "TEMPERATURE ON P LEV/UV GRID" ;" ;
+#		UM_m01s30i204_vn1106:units = "K" ;
+#		UM_m01s30i204_vn1106:cell_methods = "time: point" ;
+# with
+# 	time = 40 ;
+#	latitude_1 = 1921 ;
+#	latitude = 1920 ;
+#	longitude_1 = 2560 ;
+#	air_pressure_3 = 11 ;
+
+#  This variable is held in an 18 GB file on S3 storage
+
+from activestorage.active import Active
+import numpy as np
+from time import time
+
+S3_BUCKET = "bnl"
+S3_URL = "https://uor-aces-o.s3-ext.jc.rl.ac.uk"
+ACTIVE_URL = "https://192.171.169.248:8080"
+
+def timeseries(location='uni', blocks_MB=1, version=2, threads=100):
+
+    invoke = time()
+    storage_options = {
+        'key': "f2d55c6dcfc7618b2c34e00b58df3cef",
+        'secret': "$/'#M{0{/4rVhp%n^(XeX$q@y#&(NM3W1->~N.Q6VP.5[@bLpi='nt]AfH)>78pT",
+        'client_kwargs': {'endpoint_url': f"{S3_URL}"}, 
+            'default_fill_cache':False,
+            'default_cache_type':"readahead",
+            'default_block_size': blocks_MB * 2**20
+    }
+
+    filename = 'ch330a.pc19790301-def.nc'
+    uri = S3_BUCKET + '/' + filename
+    var = "UM_m01s30i204_vn1106"
+
+    active = Active(uri, var, storage_type="s3", max_threads=threads,
+                            storage_options=storage_options,
+                            active_storage_url=ACTIVE_URL)
+
+    # set active to use the remote reductionist.
+    # (we intend to change the invocation method
+    # to something more obvious and transparent.)
+    active._version = version
+
+    # and set the operation, again, the API will change
+    active._method = "mean"
+
+    # get hemispheric mean timeseries:
+    # (this would be more elegant in cf-python)
+    ts = []
+    md = []
+    for i in range(40):
+        ts.append(active[i,0,0:960,:][0])
+        # get some performance diagnostics from pyactive
+        print(active.metric_data)
+        if i == 0:
+            nct = active.metric_data['load nc time']
+        md.append(active.metric_data['reduction time (s)'])
+
+    result = np.array(ts)
+    print(result)
+    complete = time()
+    method = {1:'Local',2:'Active'}[version]
+    titlestring = f"{location}:{method} (T{threads},BS{blocks_MB}): {nct:.3}s,{sum(md):.4}s,{complete-invoke:.4}s"
+    print('Summary: ',titlestring)

bnl/bnl_timeseries_plot.py

@@ -0,0 +1,67 @@
+from pathlib import Path
+import numpy as np
+from matplotlib import pyplot as plt
+import os
+
+
+plt.rcParams["figure.figsize"] = (8,12)
+
+data = {'timeseries1':'Hm1, Active: 5 MB Blks',
+        'timeseries2':'Hm1, Active: 1 MB Blks',
+        'timeseries2v1':'Hm1, Local: 1 MB Blks',
+        'timeseries2-uor-t1-a':'Uni, Active: T1, 1 MB Blks',
+        'timeseries2-uor-t1-b':'Uni, Active: T1, 1 MB Blks',
+        'timeseries2-uor-t1-c':'Uni, Active: T1, 1 MB Blks',
+        'timeseries2-uor-t100-a':'Uni, Active: T100, 1 MB Blks',
+        'timeseries2-uor-t100-b':'Uni, Active: T100, 1 MB Blks',
+        'timeseries1-uor-t100-a':'Uni, Local: T100, 1 MB Blks',
+        'timeseries1-uor-t100-b':'Uni, Local: T100, 1 MB Blks',
+        'timeseries2v1-uor-t1':'Uni, Local: T1, 1 MB Blks',
+        'timeseries1-uor-t1':'Uni, Active: T1, 5 MB Blks',
+                }
+tt =  []
+
+mypath = Path(__file__).parent
+
+logfiles = mypath.glob('timeseries3-H*.log')
+
+#for d,t in data.items():
+
+for ff in logfiles:
+      
+    #fname1 = mypath/f'{d}.log'
+    #fname2 = mypath/f'{d}.metrics.txt'
+    #os.system(f'grep -a  "dataset shape" {fname1} > {fname2}')
+    #with open(fname2,'r') as f:
+    with open(ff,'r') as f:
+        lines = f.readlines()
+        dicts = [eval(l) for l in lines if l.startswith('{') ]
+        nct = dicts[0]['load nc time']
+        rt = [v['reduction time (s)'] for v in dicts]
+        summary = lines[-1][9:]
+        overall_time = float(summary.split(',')[-1][:-2])
+        tt.append((overall_time, rt, summary))
+    #os.system(f'rm {fname2}')
+
+tts = sorted(tt)
+curve = {k:v for i,v,k in tts}
+
+for k in curve.keys():
+
+    if 'Local' in k: 
+        ls = 'dashed'
+    else:
+        ls = 'solid'
+
+    plt.plot(curve[k], label=k, linestyle=ls)
+
+
+plt.legend(bbox_to_anchor=(0.7, -0.1), fontsize=8)
+plt.title('Comparison of reduction parameters')
+plt.ylabel('Time to reduce each timestep (s)')
+plt.tight_layout()
+plt.savefig('home.png')
+plt.show()
+
+
+

bnl/bnl_timeseries_runner.py

@@ -0,0 +1,15 @@
+from bnl_timeseries import timeseries
+import sys
+
+location = 'Hm1'
+blocks = [1,5]
+version = [1,2]
+threads = [1,100]
+iterations = ['a','b','c']
+
+for b in blocks:
+    for v in version:
+        for t in threads:
+            for i in iterations:
+                with open(f'timeseries3-{location}-{b}-{v}-{t}-{i}.log','w') as sys.stdout:
+                    timeseries(location, b, v, t)