Pick colab:e42a4f~..20716d: backwd comptb py3.7.12

Author: ZeyuSun

arnet/constants.py

@@ -7,9 +7,9 @@
 
 FEATURES = ['AREA', 'USFLUXL', 'MEANGBL', 'R_VALUE', 'FLARE_INDEX']
 #TODO: constants calculated on training set
-PROCESSED_DATA_DIR = '/home/zeyusun/work/flare-prediction-smarp/datasets/M_Q_24hr'
+PROCESSED_DATA_DIR = 'datasets/M_Q_24hr'
 
-@lru_cache
+@lru_cache(8)
 def get_constants():
     CONSTANTS = {}
     for dataset in ['sharp', 'smarp']:

arnet/dataset.py

@@ -96,7 +96,7 @@ def __getitem__(self, idx):
         t_end = datetime.strptime(s['t_end'], '%Y-%m-%d %H:%M:%S').strftime('%Y%m%d%H%M%S')
         largest_flare = max(s['flares'].split('|')) #WARNING: X10+
         meta = f'{idx}_{s["prefix"]}{s["arpnum"]:06d}_{t_end}_H0_W0_{largest_flare}.npy'
-        return *data_list, label, meta
+        return (*data_list, label, meta)
 
     def load_video(self, prefix, arpnum, t_now, bad_img_idx):
         t_now = drms.to_datetime(t_now)

download.py

@@ -8,8 +8,8 @@
 from tqdm import tqdm, trange
 import drms
 from sunpy.time import TimeRange
-from sunpy.instr.goes import get_goes_event_list
-
+from sunpy.net import Fido
+from sunpy.net import attrs as a
 
 ######### Change these ##########
 EMAIL = '[email protected]'
@@ -139,14 +139,27 @@ def download_smarp_images(tarpnum):
 
 
 def download_goes_per_year(year):
+    print(year)
     t_start = datetime(year=year, month=1, day=1)
     t_end = datetime(year=year+1, month=1, day=1)
-    timerange = TimeRange(t_start, t_end)
-    event_list = get_goes_event_list(timerange)
-    if len(event_list) == 0:
+    results = Fido.search(
+        a.Time(t_start, t_end),
+        a.hek.EventType("FL"),
+        # a.hek.FL.GOESCls > "M1.0",
+        a.hek.OBS.Observatory == "GOES"
+    )
+    if not results.all_colnames: # no columns / no results
         return None
 
-    event_df = pd.DataFrame(event_list)
+    event_table = results['hek']["event_starttime", "event_peaktime", "event_endtime", "fl_goescls", "ar_noaanum"]
+    event_df = event_table.to_pandas().rename(columns={
+        'event_starttime': 'start_time',
+        'event_peaktime': 'peak_time',
+        'event_endtime': 'end_time',
+        'fl_goescls': 'goes_class',
+        #'hgc_coord': 'goes_location',
+        'ar_noaanum': 'noaa_active_region',
+    })
     event_df = event_df[event_df['noaa_active_region'] != 0]
     if len(event_df) == 0:
         return None

requirements.txt

@@ -108,7 +108,7 @@ tomli==1.2.2
 torch==1.10.0
 torchinfo==1.5.3
 torchmetrics==0.6.0
-torchvision==0.11.1
+torchvision==0.11.2
 tqdm==4.62.2
 traitlets==5.0.5
 typing-extensions==3.10.0.1
@@ -121,3 +121,5 @@ yacs==0.1.8
 yarl==1.6.3
 zipp==3.6.0
 scikit-image==0.19.1
+sunpy[net]==3.1.2  # stable as of Dec 18, 2021
+torchinfo==1.5.3

run_arnet.py

@@ -126,11 +126,9 @@ def main():
             'TRAINER.default_root_dir', 'lightning_logs_dev'
         ])
 
-    with cProfile.Profile() as p:
-        mlflow.set_experiment(experiment_name=args.experiment_name)
-        with mlflow.start_run(run_name=args.run_name) as run:
-            launch(args.config, args.modes, args.resume, args.opts)
-    pstats.Stats(p).sort_stats('cumtime').print_stats(50)
+    mlflow.set_experiment(experiment_name=args.experiment_name)
+    with mlflow.start_run(run_name=args.run_name) as run:
+        launch(args.config, args.modes, args.resume, args.opts)
 
 
 def sweep():