Update readme and add pypi auto publish (#25)

* Add option to change output directory

* add pypi publish workflow

* use pypi test

* use pypi prod

* add output folder selection

* fix versioning in action

* Update package metadata and README

.github/workflows/publish.yml

@@ -0,0 +1,34 @@
+name: Upload Python Package to PyPI when a Release is Created
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  pypi-publish:
+    name: Publish release to PyPI
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/ngiab_data_preprocess
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.x"
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install setuptools wheel build
+      - name: Update package version
+        run: |
+          sed -i 's/^#version.*/version = "${{ github.event.release.tag_name }}"/g' pyproject.toml
+          sed -i '/^dynamic.*/d' pyproject.toml
+      - name: Build package
+        run: |
+          python -m build
+      - name: Publish package distributions to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

.gitignore

@@ -1,4 +1,4 @@
-conus*.*
+modules/*/conus*.*
 *.parquet
 output/*
 !output/.gitkeep

README.md

@@ -1,6 +1,6 @@
 This repository contains tools for preparing data to run a [next gen](https://github.com/NOAA-OWP/ngen) simulation using [NGIAB](https://github.com/CIROH-UA/NGIAB-CloudInfra). The tools allow you to select a catchment of interest on an interactive map, choose a date range, and prepare the data with just a few clicks!
 
-![map screenshot](https://github.com/CIROH-UA/NGIAB_data_preprocess/blob/main/map_app/static/resources/screenshot.png)
+![map screenshot](https://github.com/CIROH-UA/NGIAB_data_preprocess/blob/main/modules/map_app/static/resources/screenshot.png)
 
 
 ## What does this tool do?
@@ -15,43 +15,47 @@ This tool prepares data to run a next gen simulation by creating a run package t
 
 `sudo apt install gdal-bin` will install gdal and ogr2ogr on ubuntu / wsl
 
-## Installation
+## Installation and running
 
-To install and run the tool, follow these steps:
-
-1. Clone the repository:
-    ```bash
-    git clone https://github.com/CIROH-UA/NGIAB_data_preprocess
-    cd NGIAB_data_preprocess
-    ```
-
-2. Create a virtual environment and activate it:
-    ```bash
-    python3 -m venv env
-    source env/bin/activate
-    ```
-
-3. Install the tool:
-    ```bash
-    pip install -e .
-    ```
+```bash
+#optional but encouraged: create a virtual environment
+python3 -m venv env
+source env/bin/activate
 
-4. Run the map app:
-    ```bash
-    python -m map_app
-    ```
+# installing and running the tool
+pip install ngiab_data_preprocess
+python -m map_app
+```
 
 The first time you run this command, it will download the hydrofabric and model parameter files from Lynker Spatial. If you already have them, place `conus.gpkg` and `model_attributes.parquet` into `modules/data_sources/`.
 
-If step 4 does not work, run the following:
-
-```bash
-touch .dev
-python -m map_app
-```   
-
-This .dev file in the root folder disables the automatic browser opening so you will need to manually open [http://localhost:5000](http://localhost:5000) after running the `python -m map_app` command.   
-
+## Development Installation
+<details>
+  <summary>Installation steps</summary>
+    To install and run the tool, follow these steps:
+
+    1. Clone the repository:
+        ```bash
+        git clone https://github.com/CIROH-UA/NGIAB_data_preprocess
+        cd NGIAB_data_preprocess
+        ```
+
+    2. Create a virtual environment and activate it:
+        ```bash
+        python3 -m venv env
+        source env/bin/activate
+        ```
+
+    3. Install the tool:
+        ```bash
+        pip install -e .
+        ```
+
+    4. Run the map app:
+        ```bash
+        python -m map_app
+        ```
+</details>
 
 ## Usage
 

modules/data_processing/file_paths.py

@@ -9,6 +9,8 @@ class file_paths:
     methods do.
     """
 
+    config_file = Path("~/.NGIAB_data_preprocess").expanduser()
+
     def __init__(self, wb_id: str):
         """
         Initialize the file_paths class with a water body ID.
@@ -19,12 +21,27 @@ def __init__(self, wb_id: str):
         """
         self.wb_id = wb_id
 
+    @staticmethod
+    def get_working_dir() -> Path:
+        try:
+            with open(file_paths.config_file, "r") as f:
+                return Path(f.readline().strip()).expanduser()
+        except FileNotFoundError:
+            return None
+
+    @staticmethod
+    def set_working_dir(working_dir: Path) -> None:
+        with open(file_paths.config_file, "w") as f:
+            f.write(str(working_dir))
+
     @staticmethod
     def data_sources() -> Path:
         return Path(__file__).parent.parent / "data_sources"
 
     @staticmethod
     def root_output_dir() -> Path:
+        if file_paths.get_working_dir() is not None:
+            return file_paths.get_working_dir()
         return Path(__file__).parent.parent.parent / "output"
 
     @staticmethod

modules/map_app/__main__.py

@@ -11,18 +11,27 @@
 import requests
 import webbrowser
 
+
 def download_file(url, save_path):
     response = requests.get(url, stream=True)
-    total_size = int(response.headers.get('content-length', 0))
+    total_size = int(response.headers.get("content-length", 0))
     bytes_downloaded = 0
     chunk_size = 1048576
     with open(save_path, "wb") as f:
-        for data in tqdm(response.iter_content(chunk_size=chunk_size), total=total_size/chunk_size, unit='MB', unit_scale=True):
+        for data in tqdm(
+            response.iter_content(chunk_size=chunk_size),
+            total=total_size / chunk_size,
+            unit="MB",
+            unit_scale=True,
+        ):
             bytes_downloaded += len(data)
             f.write(data)
 
+
 hydrofabric_url = "https://lynker-spatial.s3-us-west-2.amazonaws.com/hydrofabric/v20.1/conus.gpkg"
-model_attributes_url = "https://lynker-spatial.s3-us-west-2.amazonaws.com/hydrofabric/v20.1/model_attributes.parquet"
+model_attributes_url = (
+    "https://lynker-spatial.s3-us-west-2.amazonaws.com/hydrofabric/v20.1/model_attributes.parquet"
+)
 
 
 if not file_paths.conus_hydrofabric().is_file():
@@ -44,7 +53,19 @@ def download_file(url, save_path):
     else:
         print("Exiting...")
         exit()
-
+
+if not file_paths.config_file.is_file():
+    # prompt the user to set the working directory
+    print(
+        "Output directory is not set. Would you like to set it now? Defaults to ~/ngiab_preprocess_output/ (y/N)"
+    )
+    response = input()
+    if response.lower() == "y":
+        response = input("Enter the path to the working directory: ")
+    if response == "" or response.lower() == "n":
+        response = "~/ngiab_preprocess_output/"
+    file_paths.set_working_dir(response)
+
 
 with open("app.log", "w") as f:
     f.write("")
@@ -72,12 +93,13 @@ def download_file(url, save_path):
 console_handler.setFormatter(formatter)
 logging.getLogger("").addHandler(console_handler)
 
+
 def open_browser():
     webbrowser.open("http://localhost:5000")
 
 
 if __name__ == "__main__":
-    
+
     if file_paths.dev_file().is_file():
         with open("app.log", "a") as f:
             f.write("Running in debug mode\n")

modules/map_app/static/css/main.css

@@ -83,6 +83,15 @@ button:hover {
     width: fit-content;
 }
 
+#output-info {
+    margin: 20px;
+    padding: 15px;
+    background: #ebffaf;
+    border-left: 5px solid #fdf05c;
+    border-radius: 8px;
+    width: fit-content;
+}
+
 /* Form input styling */
 input[type=datetime-local] {
     padding: 10px;

modules/map_app/templates/index.html

@@ -37,6 +37,9 @@ <h2>Selected Basins</h2>
             <button id="subset-to-file-button">Output list of upstreams to file</button>
             <img class="loading" src="{{ url_for('static', filename='resources/loading.gif') }}" alt="Loading"
                 id="subset-to-file-loading">
+            <div id="output-info">
+                <strong>Output folder is configured in the ~/.NGIAB_data_preprocess file</strong>
+            </div>
             <div id="output-path"></div>
         </section>
 

pyproject.toml

@@ -15,7 +15,7 @@ name = "ngiab_data_preprocess"
 #version = "0.0.1"
 dynamic = ["version"]
 authors = [{ name = "Josh Cunningham", email = "[email protected]" }]
-description = "A small example package"
+description = "Graphical Tools for creating Next Gen Water model input data."
 readme = "README.md"
 requires-python = ">=3.10"
 classifiers = [
@@ -51,8 +51,8 @@ dependencies = [
 ]
 
 [project.urls]
-Homepage = "https://github.com/pypa/sampleproject"
-Issues = "https://github.com/pypa/sampleproject/issues"
+Homepage = "https://github.com/CIROH-UA/NGIAB_data_preprocess"
+Issues = "https://github.com/CIROH-UA/NGIAB_data_preprocess/issues"
 
 [build-system]
 # scm adds files tracked by git to the package