Merging

open-data-rescue · Mar 2, 2024 · 12f3017 · 12f3017
2 parents 002e347 + e1f5eea
commit 12f3017
Show file tree

Hide file tree

Showing 42 changed files with 2,017 additions and 477 deletions.
diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
@@ -0,0 +1,99 @@
+---
+name: "Build a Docker image for deployment"
+
+on:
+  push:
+    branches:
+      - main
+    tags:
+      - 'v*'
+  workflow_dispatch:
+    inputs:
+      logLevel:
+        description: 'Log level'
+        required: true
+        default: 'warning'
+        type: choice
+        options:
+        - info
+        - warning
+        - debug
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+
+    # env...
+    env:
+      REGISTRY: ghcr.io
+      IMAGE_NAME: ${{ github.repository }}
+
+    steps:
+      - name: Maximize build space
+        uses: easimon/maximize-build-space@master
+        with:
+          swap-size-mb: 4096
+          temp-reserve-mb: 4096
+          root-reserve-mb: 16384
+          remove-dotnet: 'true'
+          remove-android: 'true'
+          remove-haskell: 'true'
+
+      - uses: actions/checkout@v3
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            type=ref,event=branch
+            type=ref,event=pr
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=semver,pattern=latest
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Builder instance name
+        run: echo ${{ steps.buildx.outputs.name }}
+
+      - name: Available platforms
+        run: echo ${{ steps.buildx.outputs.platforms }}
+
+      - name: Cache Docker layers
+        uses: actions/cache@v3
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-
+
+      #
+      - name: Build and Push the image
+        id: docker_push
+        uses: docker/build-push-action@v4
+        with:
+          context: ./
+          file: ./Dockerfile
+          builder: ${{ steps.buildx.outputs.name }}
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache-new
+
+      # Needed to prevent cache from growing forever (see https://github.com/docker/build-push-action/issues/252)
+      - name: Move cache
+        run: |
+          rm -rf /tmp/.buildx-cache
+          mv /tmp/.buildx-cache-new /tmp/.buildx-cache
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1,18 @@
+# See http://help.github.com/ignore-files/ for more about ignoring files.
+#
+# If you find yourself ignoring temporary files generated by your text editor
+# or operating system, you probably want to add a global ignore instead:
+#   git config --global core.excludesfile ~/.gitignore_global
+
+/log
+/tmp
+tmp/*
+rerun.txt
+*.DS_Store
+.project
+.docker-sync
+.envrc
+.vscode
+
+# Ignore pow environment settings
+.powenv
diff --git a/DRAW-post-processing/config.py b/DRAW-post-processing/config.py
@@ -4,53 +4,60 @@
 
 # assigning post-process ID's to field ID's
 # assign a TUPLE to multiple field_id's with one PPID; otherwise assign integer for single field_id
+# 1: pressure(in_hg), 2: vapour pressure(in_Hg), 3:temperature (F), 4: precipitation (in), 5: direction, 6: velocity (miles), 7: weather, 8: cloud type, 9:time, 10: RH (%), 11: cloud cover (tenths), 12 :various (character)
 ppid_to_field_id = {1: (4, 6, 7, 8, 67, 69),
                     2: 14,
-                    3: (5,9,10,11,12,13,36,37,38,39,63,64,68,76,77,78,79),
-                    4: (25, 26, 27, 28, 29, 30, 31, 50, 70, 71),
-                    5: (16, 17, 22, 23, 24, 53, 54),
-                    6: (19, 20, 21, 34, 35, 48, 61, 85),
-                    7: (18, 40, 41, 44, 51, 52, 56, 57, 66, 80, 82, 83)}
-
-sef_type_to_field_id ={ "atb": (5,68),
-                       "au": (47),
-                       "cl":(22,53),
-                       "cd":(23),
-                       "ch":(16,54),
-                       "dd":(19),
-                       "e":(14),
-                       "hd":(17),
-                       "mslp":(7),
-                       "nl":(24),
-                       "p": (4,67),
-                       "p_cor":(7,69),
-                       "pr":(27,31),
-                       "ptb":(25,28),
-                       "pte":(26,29),
-                       "rh":(15,58,59,73,75,60,72),
-                       "rrt":(66),
-                       "sd":(50),
-                       "rain_dur":(70),
-                       "snow_dur":(71),
-                       "ss":(65),
-                       "ta":9,
-                       "ta_cor":(10),
-                       "tb":(11),
-                       "tb_cor":(12),
-                       "td":(33),
-                       "TGn":(62,81),
-                       "Tn":(38,76),
-                       "Tn_cor":(37,77),
-                       "Tx":(36,78),
-                       "Tx_cor":(37,79),
-                       "Tsx":(63,64),
-                       "wf":(85),
-                       "w":(34,48,35),
-                       "ws":(20),
-                       "ww":(18,52,40,44,57,56,80,83),
-                       "w2":(51,41,82)
-                       }
-sef_type_to_unit={ "atb": "C",
+                    3: (5, 9, 10, 11, 12, 13, 33, 36, 37, 38, 39, 62, 63, 64, 68, 76, 77, 78, 79, 81),
+                    4: (27, 30, 31, 50),
+                    5: (17, 19, 23),
+                    6: (20),
+                    7: (18, 40, 41, 44, 51, 52, 56, 57, 66, 80, 82, 83),
+                    8: (16, 22, 53, 54),
+                    9: (25, 26, 28, 29, 46, 65, 70, 71),
+                    10: (15, 58, 59, 60, 72, 74, 75),
+                    11: (24),
+                    12: (21, 61, 42, 47, 66),
+                    13: (34, 35, 48)}
+
+sef_type_to_field_id = {"atb": (5, 68),
+                        "au": (47),
+                        "cl": (22, 53),
+                        "cd": (23),
+                        "ch": (16, 54),
+                        "dd": (19),
+                        "e": (14),
+                        "hd": (17),
+                        "mslp": (7),
+                        "nl": (24),
+                        "p": (4, 67),
+                        "p_cor": (7, 69),
+                        "pr": (27, 31),
+                        "ptb": (25, 28),
+                        "pte": (26, 29),
+                        "rh": (15, 58, 59, 73, 75, 60, 72),
+                        "rrt": (66),
+                        "sd": (50),
+                        "rain_dur": (70),
+                        "snow_dur": (71),
+                        "ss": (65),
+                        "ta": (9),
+                        "ta_cor": (10),
+                        "tb": (11),
+                        "tb_cor": (12),
+                        "td": (33),
+                        "TGn": (62, 81),
+                        "Tn": (38, 76),
+                        "Tn_cor": (37, 77),
+                        "Tx": (36, 78),
+                        "Tx_cor": (37, 79),
+                        "Tsx": (63, 64),
+                        "wf": (85),
+                        "w": (34, 48, 35),
+                        "ws": (20),
+                        "ww": (18, 52, 40, 44, 57, 56, 80, 83),
+                        "w2": (51, 41, 82)
+                        }
+sef_type_to_unit={"atb": "C",
                        "au": "text",
                        "cl":"lct",
                        "cd":"dir",
@@ -165,23 +172,26 @@ def possible_pressure_formats(value, for_leading_digits):
 
 temperature_min = -100.0
 temperature_max=  {
-    "5": 100, 
+    "5": 100,
     "9": 120,
     "10": 120,
-    "11": 110, 
+    "11": 110,
     "12": 110,
     "13": 110,
+    "33": 130,
     "36": 130,
     "37": 130,
     "38": 100,
     "39": 100,
+    "62": 130,
     "63": 160,
     "64": 160,
     "68": 100,
     "76": 100,
     "77": 100,
     "78": 130,
-    "79": 130
+    "79": 130,
+    "81": 130
 }
 
 
@@ -202,7 +212,7 @@ def possible_pressure_formats(value, for_leading_digits):
 temperature_air_wet_bulb=[[9,11,13],[10,12,13]] #same observation time: abs(abs(field[0]-field[1])-abs(field[2]))<air_wet_bulb_threshold
 
 # temperature fields to detect stat outliers
-temperature_stat_outliers=[9,10] 
+temperature_stat_outliers=[9,10]
 
 #define whether to display or not graphs for outliers
 temperature_plot_outliers=False

diff --git a/DRAW-post-processing/database_connection.py b/DRAW-post-processing/database_connection.py
@@ -10,23 +10,29 @@
 phase_2_errors=[]
 duplicateless=[]
 
+#
 db_user=os.environ.get('DRAW_local_db_user')
 db_passwd=os.environ.get('DRAW_local_db_pass')
 db_name=os.environ.get('DRAW_local_db_name')
-db_host=os.environ.get('DRAW_db_host')
+db_host=os.environ.get('DRAW_db_host',"127.0.0.1")
+db_port=os.environ.get('DRAW_db_port',"3306")
 
 # connection to copy of database on local machine
 conn = mysql.connector.connect(
     #####   FOLLOWING 3 VARIABLES TO BE CONFIGURED AS NECESSARY FOR LOCAL MACHINE:   #####
     user=db_user,
     password=db_passwd,
     database=db_name,
-    host=db_host
+    host=db_host,
+    port=db_port
 )
 
 cursor = conn.cursor()
 
-url = "mysql+mysqlconnector://"+db_user+":"+db_passwd+"@"+db_host+"/"+db_name
+
+# Problem? dialect+driver://username:password@host:port/database
+url = "mysql+mysqlconnector://"+db_user+":"+db_passwd+"@"+db_host+":"+db_port+"/"+db_name
+
 engine = sqlalchemy.create_engine(url)