Added debugging method to the handler

Author: caparker

ingest/handler.py

@@ -114,9 +114,14 @@ def cronhandler(event, context):
     pipeline_limit = settings.PIPELINE_LIMIT if 'pipeline_limit' not in event else event['pipeline_limit']
     realtime_limit = settings.REALTIME_LIMIT if 'realtime_limit' not in event else event['realtime_limit']
     metadata_limit = settings.METADATA_LIMIT if 'metadata_limit' not in event else event['metadata_limit']
+    fetchlogKey = event.get('fetchlogKey')
 
-    logger.info(f"Running cron job: {event['source']}, ascending: {ascending}")
+    # this is mostly for debuging and running the occasional file from the aws console
+    if fetchlogKey is not None:
+        limit = event.get('limit', 10)
+        return load_measurements_db(limit=limit, ascending=True, pattern=fetchlogKey)
 
+    logger.info(f"Running cron job: {event['source']}, ascending: {ascending}")
     # these exceptions are just a failsafe so that if something
     # unaccounted for happens we can still move on to the next
     # process. In case of this type of exception we will need to

local.py

@@ -41,20 +41,22 @@
 client = IngestClient()
 ## load all the data into the client
 client.load_keys([
-    [1, '~/Downloads/openaq-fetches/lcs-etl-pipeline/measures/lovemyair/2024-11-12/1731445632-1snpf.json', '2024-10-23']
+   # [1, '~/Downloads/openaq-fetches/lcs-etl-pipeline/measures/airgradient/2025-02-14/1739542053-5n5q.json', '2024-10-23']
+   # [1, '/home/christian/Downloads/1739444861-6bvu.json', '2025-02-13']
+    [7786652, 'lcs-etl-pipeline/measures/airgradient/2025-02-14/1739549254-h5b0m.json.gz', '2025-02-14']
 ])
 
 ## dump just the locations
-client.dump()
+#client.dump()
 
 # rollups and cached tables
 #client.process_hourly_data()
 #client.process_daily_data()
 #client.process_annual_data()
 #client.refresh_cached_tables()
 
-#client.dump_locations(False)
-#client.dump_measurements(load=False)
+client.dump_locations()
+client.dump_measurements(load=True)
 ## dump just the measurements
 # client.dump_measurements
 ## Dump both