diff --git a/LDMP/calculate_drought_vulnerability.py b/LDMP/calculate_drought_vulnerability.py index 1d5c2f61c..12dab4e93 100644 --- a/LDMP/calculate_drought_vulnerability.py +++ b/LDMP/calculate_drought_vulnerability.py @@ -18,13 +18,14 @@ import qgis.gui from qgis.core import QgsGeometry from qgis.PyQt import QtCore, QtWidgets, uic -from te_schemas.algorithms import ExecutionScript +from te_schemas.algorithms import AlgorithmRunMode, ExecutionScript from . import conf from .calculate import DlgCalculateBase from .jobs.manager import job_manager from .localexecution import drought from .logger import log +from .tasks import create_task DlgCalculateDroughtUi, _ = uic.loadUiType( str(Path(__file__).parent / "gui/DlgCalculateDrought.ui") @@ -145,7 +146,12 @@ def btn_calculate(self): self.close() - resp = job_manager.submit_remote_job(payload, self.script.id) + resp = create_task( + job_manager, + payload, + self.script.id, + AlgorithmRunMode.REMOTE, + ) if resp: main_msg = "Submitted" diff --git a/LDMP/calculate_lc.py b/LDMP/calculate_lc.py index 5fc42b913..07bc87876 100644 --- a/LDMP/calculate_lc.py +++ b/LDMP/calculate_lc.py @@ -22,6 +22,7 @@ from . import calculate, lc_setup from .jobs.manager import job_manager +from .tasks import create_task DlgCalculateLcUi, _ = uic.loadUiType( str(Path(__file__).parent / "gui/DlgCalculateLC.ui") @@ -97,9 +98,17 @@ def calculate_on_GEE(self): "task_name": self.execution_name_le.text(), "task_notes": self.task_notes.toPlainText(), } - job = job_manager.submit_remote_job(payload, self.script.id) - if job is not None: + create_task( + job_manager, + payload, + self.script.id, + AlgorithmRunMode.REMOTE, + self.job_submitted, + ) + + def job_submitted(self, exception, result=None): + if result is not None: main_msg = "Submitted" description = "Land cover task submitted to Trends.Earth server." else: @@ -206,4 +215,11 @@ def calculate_locally(self): "legend_nesting": initial_nesting, "trans_matrix": LCTransitionDefinitionDeg.Schema().dumps(trans_matrix), } - job_manager.submit_local_job(job_params, self.LOCAL_SCRIPT_NAME, self.aoi) + + create_task( + job_manager, + job_params, + self.LOCAL_SCRIPT_NAME, + AlgorithmRunMode.LOCAL, + self.aoi, + ) diff --git a/LDMP/calculate_ldn.py b/LDMP/calculate_ldn.py index 98472082a..812625bea 100644 --- a/LDMP/calculate_ldn.py +++ b/LDMP/calculate_ldn.py @@ -20,7 +20,7 @@ import te_algorithms.gdal.land_deg.config as ld_config from qgis.core import QgsGeometry from qgis.PyQt import QtCore, QtWidgets, uic -from te_schemas.algorithms import ExecutionScript +from te_schemas.algorithms import AlgorithmRunMode, ExecutionScript from te_schemas.land_cover import LCLegendNesting, LCTransitionDefinitionDeg from te_schemas.productivity import ProductivityMode @@ -29,6 +29,7 @@ from .jobs.manager import job_manager from .localexecution import ldn from .logger import log +from .tasks import create_task DlgCalculateOneStepUi, _ = uic.loadUiType( str(Path(__file__).parent / "gui/DlgCalculateOneStep.ui") @@ -623,7 +624,9 @@ def btn_calculate(self): self.close() for payload in payloads: - resp = job_manager.submit_remote_job(payload, self.script.id) + resp = create_task( + job_manager, payload, self.script.id, AlgorithmRunMode.REMOTE + ) if resp: main_msg = "Submitted" @@ -1212,7 +1215,13 @@ def btn_calculate(self): # # self.close() # - # resp = job_manager.submit_remote_job(payload, self.script.id) + + # resp = create_task( + # job_manager, + # payload, + # self.script.id, + # AlgorithmRunMode.REMOTE, + # ) # # if resp: # main_msg = "Submitted" diff --git a/LDMP/calculate_prod.py b/LDMP/calculate_prod.py index 39339e8b5..7821908e5 100644 --- a/LDMP/calculate_prod.py +++ b/LDMP/calculate_prod.py @@ -17,12 +17,13 @@ import qgis.gui from qgis.PyQt import QtCore, QtWidgets, uic -from te_schemas.algorithms import ExecutionScript +from te_schemas.algorithms import AlgorithmRunMode, ExecutionScript from te_schemas.productivity import ProductivityMode from . import calculate, conf from .jobs.manager import job_manager from .logger import log +from .tasks import create_task DlgCalculateProdUi, _ = uic.loadUiType( str(Path(__file__).parent / "gui/DlgCalculateProd.ui") @@ -270,7 +271,12 @@ def btn_calculate(self): else: raise ValueError("Unknown prod_mode {prod_mode}") - resp = job_manager.submit_remote_job(payload, self.script.id) + resp = create_task( + job_manager, + payload, + self.script.id, + AlgorithmRunMode.REMOTE, + ) if resp: main_msg = "Submitted" diff --git a/LDMP/calculate_rest_biomass.py b/LDMP/calculate_rest_biomass.py index fad355768..82304a2cc 100644 --- a/LDMP/calculate_rest_biomass.py +++ b/LDMP/calculate_rest_biomass.py @@ -17,11 +17,12 @@ import qgis.core import qgis.gui from qgis.PyQt import QtWidgets, uic -from te_schemas.algorithms import ExecutionScript +from te_schemas.algorithms import AlgorithmRunMode, ExecutionScript from . import calculate, data_io from .jobs.manager import job_manager from .localexecution import biomassrestoration +from .tasks import create_task DlgCalculateRestBiomassDataUi, _ = uic.loadUiType( str(Path(__file__).parent / "gui/DlgCalculateRestBiomassData.ui") @@ -79,7 +80,12 @@ def calculate_on_GEE(self): "task_notes": self.task_notes.toPlainText(), } - resp = job_manager.submit_remote_job(payload, self.script.id) + resp = create_task( + job_manager, + payload, + self.script.id, + AlgorithmRunMode.REMOTE, + ) if resp: main_msg = "Submitted" description = "Restoration biomass change submitted to Trends.Earth server." diff --git a/LDMP/calculate_soc.py b/LDMP/calculate_soc.py index 7c6d9e81b..5e9bff512 100644 --- a/LDMP/calculate_soc.py +++ b/LDMP/calculate_soc.py @@ -23,6 +23,7 @@ from .jobs.manager import job_manager from .lc_setup import get_trans_matrix from .logger import log +from .tasks import create_task DlgCalculateSocUi, _ = uic.loadUiType( str(Path(__file__).parent / "gui/DlgCalculateSOC.ui") @@ -286,7 +287,12 @@ def calculate_on_GEE(self): "task_notes": self.options_tab.task_notes.toPlainText(), } - resp = job_manager.submit_remote_job(payload, self.script.id) + resp = create_task( + job_manager, + payload, + self.script.id, + AlgorithmRunMode.REMOTE, + ) if resp: main_msg = "Submitted" description = "Soil organic carbon task submitted to Trends.Earth server." diff --git a/LDMP/calculate_tc.py b/LDMP/calculate_tc.py index a859a06e4..908a3506b 100644 --- a/LDMP/calculate_tc.py +++ b/LDMP/calculate_tc.py @@ -20,13 +20,14 @@ import qgis.gui from osgeo import gdal, osr from qgis.PyQt import QtCore, QtWidgets, uic -from te_schemas.algorithms import ExecutionScript +from te_schemas.algorithms import AlgorithmRunMode, ExecutionScript from te_schemas.schemas import BandInfo from . import GetTempFilename, calculate, conf, data_io, worker from .jobs.manager import job_manager from .logger import log from .summary import calc_cell_area +from .tasks import create_task DlgCalculateTcDataUi, _ = uic.loadUiType( str(Path(__file__).parent / "gui/DlgCalculateTCData.ui") @@ -427,7 +428,14 @@ def calculate_on_GEE(self, method, biomass_data): "task_name": self.execution_name_le.text(), "task_notes": self.task_notes.toPlainText(), } - resp = job_manager.submit_remote_job(payload, self.script.id) + + resp = create_task( + job_manager, + payload, + self.script.id, + AlgorithmRunMode.REMOTE, + ) + if resp: main_msg = "Submitted" description = "Total carbon submitted to Trends.Earth server." diff --git a/LDMP/calculate_unccd.py b/LDMP/calculate_unccd.py index f4f396175..3cf117cb7 100644 --- a/LDMP/calculate_unccd.py +++ b/LDMP/calculate_unccd.py @@ -17,13 +17,14 @@ import qgis.gui from qgis.PyQt import QtCore, QtWidgets, uic -from te_schemas.algorithms import ExecutionScript +from te_schemas.algorithms import AlgorithmRunMode, ExecutionScript from . import conf from .calculate import DlgCalculateBase from .jobs.manager import job_manager from .localexecution import unccd from .logger import log +from .tasks import create_task DlgCalculateUNCCDUi, _ = uic.loadUiType( str(Path(__file__).parent / "gui/DlgCalculateUNCCD.ui") @@ -135,7 +136,12 @@ def btn_calculate(self): self.close() - resp = job_manager.submit_remote_job(payload, self.script.id) + resp = create_task( + job_manager, + payload, + self.script.id, + AlgorithmRunMode.REMOTE, + ) if resp: main_msg = "Submitted" diff --git a/LDMP/calculate_urban.py b/LDMP/calculate_urban.py index 9af0b065a..a3a092e4c 100644 --- a/LDMP/calculate_urban.py +++ b/LDMP/calculate_urban.py @@ -17,11 +17,12 @@ import qgis.core import qgis.gui from qgis.PyQt import QtCore, QtWidgets, uic -from te_schemas.algorithms import ExecutionScript +from te_schemas.algorithms import AlgorithmRunMode, ExecutionScript from . import calculate, data_io from .jobs.manager import job_manager from .logger import log +from .task import create_task DlgCalculateUrbanDataUi, _ = uic.loadUiType( str(Path(__file__).parent / "gui/DlgCalculateUrbanData.ui") @@ -196,7 +197,12 @@ def calculate_on_GEE(self): "task_notes": self.options_tab.task_notes.toPlainText(), } - resp = job_manager.submit_remote_job(payload, self.script.id) + resp = create_task( + job_manager, + payload, + self.script.id, + AlgorithmRunMode.REMOTE, + ) if resp: main_msg = "Submitted" diff --git a/LDMP/download_data.py b/LDMP/download_data.py index ff409e081..27c1ce99a 100644 --- a/LDMP/download_data.py +++ b/LDMP/download_data.py @@ -17,12 +17,13 @@ import qgis.gui from qgis.PyQt import QtCore, QtGui, QtWidgets, uic -from te_schemas.algorithms import ExecutionScript +from te_schemas.algorithms import AlgorithmRunMode, ExecutionScript from . import calculate, conf from .conf import Setting, settings_manager from .jobs.manager import job_manager from .logger import log +from .tasks import create_task DlgDownloadUi, _ = uic.loadUiType(str(Path(__file__).parent / "gui/DlgDownload.ui")) @@ -271,7 +272,13 @@ def btn_calculate(self): "task_notes": "", } - resp = job_manager.submit_remote_job(payload, self.script.id) + resp = create_task( + job_manager, + payload, + self.script.id, + AlgorithmRunMode.REMOTE, + ) + if resp: main_msg = "Success" description = "Download request submitted to Trends.Earth server." diff --git a/LDMP/jobs/models.py b/LDMP/jobs/models.py index 7c09d010a..d3a959e52 100644 --- a/LDMP/jobs/models.py +++ b/LDMP/jobs/models.py @@ -69,7 +69,7 @@ def set_script_name_version(self, data, **kwargs): log(f"Failed to get script by id for {script_id}") script = get_job_local_script(data["script_name"]) else: - log(f"Failed to get script by id for {script_id}") + log(f"Got script from id {script_id}") script = ExecutionScript( script_id, run_mode=AlgorithmRunMode.NOT_APPLICABLE ) diff --git a/LDMP/landpks.py b/LDMP/landpks.py index 61f0cc48b..0ab78c0c7 100644 --- a/LDMP/landpks.py +++ b/LDMP/landpks.py @@ -16,11 +16,12 @@ import qgis.gui from qgis.PyQt import QtCore, QtGui, QtWidgets, uic -from te_schemas.algorithms import ExecutionScript +from te_schemas.algorithms import AlgorithmRunMode, ExecutionScript from . import calculate, conf from .jobs.manager import job_manager from .logger import log +from .tasks import create_task DlgLandPKSDownloadUi, _ = uic.loadUiType( str(Path(__file__).parent / "gui/DlgLandPKSDownload.ui") @@ -270,7 +271,13 @@ def btn_calculate(self): "task_notes": self.options_tab.task_notes.toPlainText(), } - resp = job_manager.submit_remote_job(payload, self.script.id) + resp = create_task( + job_manager, + payload, + self.script.id, + AlgorithmRunMode.REMOTE, + ) + if resp: main_msg = "Success" description = "Download request submitted to LandPKS." diff --git a/LDMP/tasks.py b/LDMP/tasks.py new file mode 100644 index 000000000..4039e8f66 --- /dev/null +++ b/LDMP/tasks.py @@ -0,0 +1,48 @@ +from qgis.core import QgsApplication, QgsTask +from te_schemas.algorithms import AlgorithmRunMode + + +def submit_job(job_manager, payload, script_id, job_type, aoi, task): + """Function that submits a job and runs in a task.""" + try: + if job_type == AlgorithmRunMode.REMOTE: + job = job_manager.submit_remote_job(payload, script_id) + elif job_type == AlgorithmRunMode.LOCAL: + job = job_manager.submit_local_job(payload, script_id, aoi) + + return job + except Exception: + return None + + +def create_task( + job_manager, payload, script_id, job_type, task_finished=None, aoi=None +): + # Create a task using fromFunction + if task_finished is None: + task_finished = on_task_finished + + task = QgsTask.fromFunction( + "Submit Job", + submit_job, + on_finished=task_finished, + job_manager=job_manager, + payload=payload, + script_id=script_id, + job_type=job_type, + aoi=aoi, + ) + + QgsApplication.taskManager().addTask(task) + + return task + + +def on_task_finished(exception, result=None): + if exception is None: + if result: + print(f"Task completed successfully, job: {result}") + else: + print("Task completed but no job returned.") + else: + print(f"Task failed: {exception}") diff --git a/LDMP/timeseries.py b/LDMP/timeseries.py index 8f3c704ac..d358375f3 100644 --- a/LDMP/timeseries.py +++ b/LDMP/timeseries.py @@ -17,7 +17,7 @@ import qgis.gui from qgis.PyQt import QtWidgets, uic from qgis.PyQt.QtCore import QDate, Qt -from te_schemas.algorithms import ExecutionScript +from te_schemas.algorithms import AlgorithmRunMode, ExecutionScript from .calculate import ( DlgCalculateBase, @@ -26,6 +26,7 @@ from .jobs.manager import job_manager from .logger import log from .settings import AreaWidget, AreaWidgetSection +from .tasks import create_task Ui_DlgTimeseries, _ = uic.loadUiType( str(Path(__file__).parent / "gui/DlgTimeseries.ui") @@ -312,7 +313,12 @@ def btn_calculate(self): ][self.traj_indic.currentText()]["params"] ) - resp = job_manager.submit_remote_job(payload, self.script.id) + resp = create_task( + job_manager, + payload, + self.script.id, + AlgorithmRunMode.REMOTE, + ) self.reset_widgets() diff --git a/gee/landpks/landpks_landtrend_plot.ipynb b/gee/landpks/landpks_landtrend_plot.ipynb index a5179bf97..8eadcc375 100644 --- a/gee/landpks/landpks_landtrend_plot.ipynb +++ b/gee/landpks/landpks_landtrend_plot.ipynb @@ -9,17 +9,18 @@ "import json\n", "\n", "import ee\n", - "\n", - "from PIL import Image\n", - "import numpy as np\n", - "import matplotlib.pyplot as plt\n", "import matplotlib.gridspec as gridspec\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", "import pandas as pd\n", + "from PIL import Image\n", "\n", - "EE_ACCOUNT = 'gef-ldmp-server@gef-ld-toolbox.iam.gserviceaccount.com'\n", + "EE_ACCOUNT = \"gef-ldmp-server@gef-ld-toolbox.iam.gserviceaccount.com\"\n", "\n", "# The private key associated with your service account in JSON format.\n", - "EE_PRIVATE_KEY_FILE = 'C:/Users/azvoleff/Code/LandDegradation/decisiontheater/te_key.json'\n", + "EE_PRIVATE_KEY_FILE = (\n", + " \"C:/Users/azvoleff/Code/LandDegradation/decisiontheater/te_key.json\"\n", + ")\n", "\n", "EE_CREDENTIALS = ee.ServiceAccountCredentials(EE_ACCOUNT, EE_PRIVATE_KEY_FILE)\n", "ee.Initialize(EE_CREDENTIALS)\n", diff --git a/gee/landpks/landpks_test_api.ipynb b/gee/landpks/landpks_test_api.ipynb index c26972283..d46df3351 100644 --- a/gee/landpks/landpks_test_api.ipynb +++ b/gee/landpks/landpks_test_api.ipynb @@ -28,11 +28,11 @@ ], "source": [ "import json\n", - "import requests\n", "import time\n", "from getpass import getpass\n", "\n", "import ipyplot\n", + "import requests\n", "from IPython.display import Image" ] }, @@ -66,7 +66,7 @@ ], "source": [ "api_url = \"https://api.trends.earth\"\n", - "#api_url = \"http://localhost:3000\"\n", + "# api_url = \"http://localhost:3000\"\n", "email, password = input(\"Email: \"), getpass(\"Password: \")\n", "creds = {\"email\": email, \"password\": password}\n", "auth_url = api_url + \"/auth\"\n",