forked from erdc/spt_compute
-
Notifications
You must be signed in to change notification settings - Fork 0
/
autorapid_process.py
202 lines (186 loc) · 12.1 KB
/
autorapid_process.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
# -*- coding: utf-8 -*-
##
## autorapid_process.py
## spt_ecmwf_autorapid_process
##
## Created by Alan D. Snow.
## Copyright © 2015-2016 Alan D Snow. All rights reserved.
## License: BSD-3 Clause
from glob import glob
import os
from geoserver.catalog import FailedRequestError as geo_cat_FailedRequestError
#local imports
from imports.helper_functions import (get_valid_watershed_list,
get_watershed_subbasin_from_folder)
#package imports
from AutoRoutePy.run_autoroute_multicore import run_autoroute_multicore
from AutoRoutePy.post_process import get_shapefile_layergroup_bounds, rename_shapefiles
from spt_dataset_manager.dataset_manager import GeoServerDatasetManager
#----------------------------------------------------------------------------------------
# MAIN PROCESS
#----------------------------------------------------------------------------------------
def run_autorapid_process(autoroute_executable_location, #location of AutoRoute executable
autoroute_io_files_location, #path to AutoRoute input/outpuf directory
rapid_io_files_location, #path to AutoRoute input/outpuf directory
forecast_date_timestep,
condor_log_directory,
geoserver_url='',
geoserver_username='',
geoserver_password='',
app_instance_id=''
):
"""
This it the main AutoRoute-RAPID process
"""
#initialize HTCondor Directory
condor_init_dir = os.path.join(condor_log_directory, forecast_date_timestep)
try:
os.makedirs(condor_init_dir)
except OSError:
pass
#run autorapid for each watershed
autoroute_watershed_jobs = {}
#get most recent forecast date/timestep
print "Running AutoRoute process for forecast:", forecast_date_timestep
#loop through input watershed folders
autoroute_input_folder = os.path.join(autoroute_io_files_location, "input")
autoroute_output_folder = os.path.join(autoroute_io_files_location, "output")
autoroute_input_directories = get_valid_watershed_list(autoroute_input_folder)
for autoroute_input_directory in autoroute_input_directories:
watershed, subbasin = get_watershed_subbasin_from_folder(autoroute_input_directory)
#RAPID file paths
master_watershed_rapid_input_directory = os.path.join(rapid_io_files_location, "input", autoroute_input_directory)
master_watershed_rapid_output_directory = os.path.join(rapid_io_files_location, 'output',
autoroute_input_directory, forecast_date_timestep)
if not os.path.exists(master_watershed_rapid_input_directory):
print "AutoRoute watershed", autoroute_input_directory, "not in RAPID IO folder. Skipping ..."
continue
if not os.path.exists(master_watershed_rapid_output_directory):
print "AutoRoute watershed", autoroute_input_directory, "missing RAPID forecast folder. Skipping ..."
continue
#setup the output location
master_watershed_autoroute_output_directory = os.path.join(autoroute_output_folder,
autoroute_input_directory,
forecast_date_timestep)
try:
os.makedirs(master_watershed_autoroute_output_directory)
except OSError:
pass
#loop through sub-directories
autoroute_watershed_directory_path = os.path.join(autoroute_input_folder, autoroute_input_directory)
autoroute_watershed_jobs[autoroute_input_directory] = run_autoroute_multicore(autoroute_executable_location, #location of AutoRoute executable
autoroute_input_directory=autoroute_watershed_directory_path, #path to AutoRoute input directory
autoroute_output_directory=master_watershed_autoroute_output_directory, #path to AutoRoute output directory
rapid_output_directory=master_watershed_rapid_output_directory, #path to ECMWF RAPID input/output directory
mode="htcondor", #multiprocess or htcondor
condor_log_directory=condor_init_dir,
wait_for_all_processes_to_finish=False
)
geoserver_manager = None
if geoserver_url and geoserver_username and geoserver_password and app_instance_id:
try:
geoserver_manager = GeoServerDatasetManager(geoserver_url,
geoserver_username,
geoserver_password,
app_instance_id)
except Exception as ex:
print ex
print "Skipping geoserver upload ..."
geoserver_manager = None
pass
#wait for jobs to finish by watershed
for autoroute_watershed_directory, autoroute_watershed_job in autoroute_watershed_jobs.iteritems():
master_watershed_autoroute_output_directory = os.path.join(autoroute_output_folder,
autoroute_watershed_directory,
forecast_date_timestep)
#time stamped layer name
geoserver_layer_group_name = "%s-floodmap-%s" % (autoroute_watershed_directory,
forecast_date_timestep)
geoserver_resource_list = []
upload_shapefile_list = []
for job_index, job_handle in enumerate(autoroute_watershed_job['htcondor_job_list']):
job_handle.wait()
#time stamped layer name
geoserver_resource_name = "%s-%s" % (geoserver_layer_group_name,
job_index)
#upload each shapefile
upload_shapefile = os.path.join(master_watershed_autoroute_output_directory,
"%s%s" % (geoserver_resource_name, ".shp"))
#rename files
rename_shapefiles(master_watershed_autoroute_output_directory,
os.path.splitext(upload_shapefile)[0],
autoroute_watershed_job['htcondor_job_info'][job_index]['output_shapefile_base_name'])
#upload to GeoServer
if geoserver_manager:
if os.path.exists(upload_shapefile):
upload_shapefile_list.append(upload_shapefile)
print "Uploading", upload_shapefile, "to GeoServer as", geoserver_resource_name
shapefile_basename = os.path.splitext(upload_shapefile)[0]
#remove past layer if exists
#geoserver_manager.purge_remove_geoserver_layer(geoserver_manager.get_layer_name(geoserver_resource_name))
#upload updated layer
shapefile_list = glob("%s*" % shapefile_basename)
#Note: Added try, except statement because the request search fails when the app
#deletes the layer after request is made (happens hourly), so the process may throw
#an exception even though it was successful.
"""
...
File "/home/alan/work/scripts/spt_ecmwf_autorapid_process/spt_dataset_manager/dataset_manager.py", line 798, in upload_shapefile
overwrite=True)
File "/usr/lib/tethys/local/lib/python2.7/site-packages/tethys_dataset_services/engines/geoserver_engine.py", line 1288, in create_shapefile_resource
new_resource = catalog.get_resource(name=name, workspace=workspace)
File "/usr/lib/tethys/local/lib/python2.7/site-packages/geoserver/catalog.py", line 616, in get_resource
resource = self.get_resource(name, store)
File "/usr/lib/tethys/local/lib/python2.7/site-packages/geoserver/catalog.py", line 606, in get_resource
candidates = [s for s in self.get_resources(store) if s.name == name]
File "/usr/lib/tethys/local/lib/python2.7/site-packages/geoserver/catalog.py", line 645, in get_resources
return store.get_resources()
File "/usr/lib/tethys/local/lib/python2.7/site-packages/geoserver/store.py", line 58, in get_resources
xml = self.catalog.get_xml(res_url)
File "/usr/lib/tethys/local/lib/python2.7/site-packages/geoserver/catalog.py", line 188, in get_xml
raise FailedRequestError("Tried to make a GET request to %s but got a %d status code: \n%s" % (rest_url, response.status, content))
geoserver.catalog.FailedRequestError: ...
"""
try:
geoserver_manager.upload_shapefile(geoserver_resource_name,
shapefile_list)
except geo_cat_FailedRequestError as ex:
print ex
print "Most likely OK, but always wise to check ..."
pass
geoserver_resource_list.append(geoserver_manager.get_layer_name(geoserver_resource_name))
#TODO: Upload to CKAN for history of predicted floodmaps?
else:
print upload_shapefile, "not found. Skipping upload to GeoServer ..."
if geoserver_manager and geoserver_resource_list:
print "Creating Layer Group:", geoserver_layer_group_name
style_list = ['green' for i in range(len(geoserver_resource_list))]
bounds = get_shapefile_layergroup_bounds(upload_shapefile_list)
geoserver_manager.dataset_engine.create_layer_group(layer_group_id=geoserver_manager.get_layer_name(geoserver_layer_group_name),
layers=tuple(geoserver_resource_list),
styles=tuple(style_list),
bounds=tuple(bounds))
#remove local shapefile when done
for upload_shapefile in upload_shapefile_list:
shapefile_parts = glob("%s*" % os.path.splitext(upload_shapefile)[0])
for shapefile_part in shapefile_parts:
try:
os.remove(shapefile_part)
except OSError:
pass
#remove local directories when done
try:
os.rmdir(master_watershed_autoroute_output_directory)
except OSError:
pass
if __name__ == "__main__":
run_autorapid_process(autoroute_executable_location='/home/alan/work/scripts/AutoRoute/source_code/autoroute',
autoroute_io_files_location='/home/alan/work/autoroute-io',
rapid_io_files_location='/home/alan/work/rapid-io',
forecast_date_timestep='20151217.0',
condor_log_directory='/home/alan/work/condor/',
#geoserver_url=',
#geoserver_username='',
#geoserver_password='',
#app_instance_id='',
)