Fix multiprocessing and consolidate QC

regridding/README.md

@@ -1,6 +1,8 @@
 # Regrid CMIP6 on ACDN
 
-This directory is used for regridding the CMIP6 data mirrored on the ACDN to a common grid. The target grid is the NCAR CESM2 grid, which is also shared by a number of the other chosen models.
+This directory is used for regridding the CMIP6 data mirrored on the Arctic Climate Data Node. This pipeline will regrid the specified set of models, scenarios, variables, and frequencies (e.g. temporal resolutions) to the grid of some specified file (referred to as the target grid file). This includes cropping the extent of regridded outputs to match the extent of the target file if it is of a larger extent.
+
+Note - this pipeline was previously used for a single fixed grid - that used by the NCAR CESM2 model, which is also shared by a few of the other chosen models (TaiESM1, NorESM2-MM).
 
 This pipeline also crops these datasets to a pan-arctic domain of 50N - 90N. 
 

regridding/config.py

@@ -1,6 +1,7 @@
 """Lookup tables for CMIP6 regridding. 
 Since we are providing our config via Prefect, these were copied from the transfers/config.py file
-to avoid using system environment variables."""
+to avoid using system environment variables.
+"""
 
 # batch file naming template
 batch_tmp_fn = "batch_{model}_{scenario}_{frequency}_{var_id}_{grid_name}_{count}.txt"
@@ -14,23 +15,6 @@
     "ssp585",
 ]
 
-# institution model strings (<institution>_<model>, from mirrored data) that we will be regridding
-inst_models = [
-    "NOAA-GFDL_GFDL-ESM4",
-    "NIMS-KMA_KACE-1-0-G",
-    "CNRM-CERFACS_CNRM-CM6-1-HR",
-    "NCC_NorESM2-MM",
-    "AS-RCEC_TaiESM1",
-    "MOHC_HadGEM3-GC31-MM",
-    "MOHC_HadGEM3-GC31-LL",
-    "MIROC_MIROC6",
-    "EC-Earth-Consortium_EC-Earth3-Veg",
-    "NCAR_CESM2",
-    "MPI-M_MPI-ESM1-2-HR",
-    "DKRZ_MPI-ESM1-2-HR",
-    "MRI_MRI-ESM2-0",
-]
-
 model_inst_lu = {
     "ACCESS-CM2": "CSIRO-ARCCSS",
     "CESM2": "NCAR",
@@ -46,11 +30,8 @@
     "TaiESM1": "AS-RCEC",
     "CESM2-WACCM": "NCAR",
     # Another oddity - MPI-ESM1-2-* models have different representation among the institutions, or "Institution ID".
-    # the -HR version is apparently mostly available under "DKRZ". The -LR version is mostly available under "MPI-M".
-    # There is apparently mixing, too, as the -HR version has historical data under "MPI-M", and the -LR version has
-    #  data available under "DKRZ". We will just go with the institution which has the majority for each, for now.
+    # the -HR version was run by "DKRZ" for ScenarioMIP data and MPI-M for CMIP experiment
     "MPI-ESM1-2-HR": "DKRZ",
-    "MPI-ESM1-2-LR": "MPI-M",
 }
 
 variables = {
@@ -93,8 +74,8 @@
     },
     "prsn": {
         "name": "snowfall_flux",
-        "table_ids": ["Amon", "Omon", "day"],
-    },  # some models use Omon for table ID
+        "table_ids": ["Amon", "day"],
+    },
     "snd": {"name": "surface_snow_thickness", "table_ids": ["LImon", "Eday"]},
     "snw": {"name": "surface_snow_amount", "table_ids": ["LImon", "day"]},
     "rlds": {
@@ -134,3 +115,28 @@
         "table_ids": ["Amon", "day", "Eday"],
     },
 }
+
+landsea_variables = {
+    "mrro": "land",
+    "mrsos": "land",
+    "mrsol": "land",
+    "snd": "land",
+    "snw": "land",
+    "siconc": "sea",
+}
+
+# lookup for the sftlf file paths for each model, hardcoded paths for now
+model_sftlf_lu = {
+    "GFDL-ESM4": "/beegfs/CMIP6/arctic-cmip6/CMIP6/ScenarioMIP/NOAA-GFDL/GFDL-ESM4/ssp370/r1i1p1f1/fx/sftlf/gr1/v20180701/sftlf_fx_GFDL-ESM4_ssp370_r1i1p1f1_gr1.nc",
+    "CNRM-CM6-1-HR": "/beegfs/CMIP6/arctic-cmip6/CMIP6/CMIP/CNRM-CERFACS/CNRM-CM6-1-HR/historical/r1i1p1f2/fx/sftlf/gr/v20191021/sftlf_fx_CNRM-CM6-1-HR_historical_r1i1p1f2_gr.nc",
+    "NorESM2-MM": "/beegfs/CMIP6/arctic-cmip6/CMIP6/CMIP/NCC/NorESM2-MM/historical/r1i1p1f1/fx/sftlf/gn/v20191108/sftlf_fx_NorESM2-MM_historical_r1i1p1f1_gn.nc",
+    "TaiESM1": "/beegfs/CMIP6/arctic-cmip6/CMIP6/CMIP/AS-RCEC/TaiESM1/historical/r1i1p1f1/fx/sftlf/gn/v20200624/sftlf_fx_TaiESM1_historical_r1i1p1f1_gn.nc",
+    "HadGEM3-GC31-MM": "/beegfs/CMIP6/arctic-cmip6/CMIP6/CMIP/MOHC/HadGEM3-GC31-MM/piControl/r1i1p1f1/fx/sftlf/gn/v20200108/sftlf_fx_HadGEM3-GC31-MM_piControl_r1i1p1f1_gn.nc",
+    "HadGEM3-GC31-LL": "/beegfs/CMIP6/arctic-cmip6/CMIP6/CMIP/MOHC/HadGEM3-GC31-LL/piControl/r1i1p1f1/fx/sftlf/gn/v20190709/sftlf_fx_HadGEM3-GC31-LL_piControl_r1i1p1f1_gn.nc",
+    "MIROC6": "/beegfs/CMIP6/arctic-cmip6/CMIP6/CMIP/MIROC/MIROC6/historical/r1i1p1f1/fx/sftlf/gn/v20190311/sftlf_fx_MIROC6_historical_r1i1p1f1_gn.nc",
+    "EC-Earth3-Veg": "/beegfs/CMIP6/arctic-cmip6/CMIP6/CMIP/EC-Earth-Consortium/EC-Earth3-Veg/historical/r1i1p1f1/fx/sftlf/gr/v20211207/sftlf_fx_EC-Earth3-Veg_historical_r1i1p1f1_gr.nc",
+    "CESM2": "/beegfs/CMIP6/arctic-cmip6/CMIP6/CMIP/NCAR/CESM2/historical/r11i1p1f1/fx/sftlf/gn/v20190514/sftlf_fx_CESM2_historical_r11i1p1f1_gn.nc",
+    "MPI-ESM1-2-HR": "/beegfs/CMIP6/arctic-cmip6/CMIP6/CMIP/MPI-M/MPI-ESM1-2-HR/historical/r1i1p1f1/fx/sftlf/gn/v20190710/sftlf_fx_MPI-ESM1-2-HR_historical_r1i1p1f1_gn.nc",
+    "MRI-ESM2-0": "/beegfs/CMIP6/arctic-cmip6/CMIP6/CMIP/MRI/MRI-ESM2-0/historical/r1i1p1f1/fx/sftlf/gn/v20190603/sftlf_fx_MRI-ESM2-0_historical_r1i1p1f1_gn.nc",
+    # no sftlf files for E3SM models or KACE-1-0-G
+}