Merge branch '1.3.3' of https://github.com/nasa-gibs/onearth into 2.1.1

# Conflicts:
#	deploy/onearth/onearth.spec
#	doc/config_reproject.md
#	src/layer_config/bin/oe_configure_layer.py
#	src/layer_config/layers/layer_configuration_file_reproject.xml.sample
#	src/layer_config/mapserver/EPSG3857.header
#	src/modules/mod_oems/Makefile
#	src/modules/mod_oems/README.md
#	src/modules/mod_oems/mod_oems.cpp
#	src/modules/mod_oems/mod_oems.h
#	src/modules/mod_reproject
#	src/scripts/oe_configure_reproject_layer.py
#	src/test/layer_config_files/config_templates/test_mapfile_layer_contents.map
#	src/test/layer_config_files/config_templates/test_vector_mapfile_layer_contents_inclusion.xml
#	src/test/layer_config_files/config_templates/test_vector_mapfile_type_inclusion.xml
#	src/test/test_configure_layer.py
#	src/test/test_mod_oems.py

src/test/oe_test_utils.py

@@ -246,14 +246,15 @@ def add_trailing_slash(directory_path):
 
 
 def restart_apache():
-    try:
-        check_apache_running()
-        if "el7" in platform.release():
-            apache = subprocess.Popen('pkill --signal HUP --uid root httpd'.split(), stderr=subprocess.STDOUT, stdout=subprocess.PIPE)
-        else:
-            apache = subprocess.Popen(['apachectl', 'restart'], stderr=subprocess.STDOUT, stdout=subprocess.PIPE)
-    except ValueError:
-        apache = subprocess.Popen(['httpd'], stderr=subprocess.STDOUT, stdout=subprocess.PIPE)
+    apache = subprocess.Popen(['httpd', '-k', 'restart'], stderr=subprocess.STDOUT, stdout=subprocess.PIPE)
+    # try:
+    #     check_apache_running()
+    #     if "el7" in platform.release():
+    #         apache = subprocess.Popen('pkill --signal HUP --uid root httpd'.split(), stderr=subprocess.STDOUT, stdout=subprocess.PIPE)
+    #     else:
+    #         apache = subprocess.Popen(['apachectl', 'restart'], stderr=subprocess.STDOUT, stdout=subprocess.PIPE)
+    # except ValueError:
+    #     apache = subprocess.Popen(['httpd'], stderr=subprocess.STDOUT, stdout=subprocess.PIPE)
     (stdout, stderr) = apache.communicate()
     if stdout != None and len(stdout) != 0:
         sys.stderr.write("\n=== STDOUT from restart_apache():\n%s\n===\n" % stdout.rstrip())
@@ -577,7 +578,7 @@ def get_layer_config(filepath, archive_config):
         pass
     try:
         config['vector_type'] = config_dom.getElementsByTagName('VectorType')[0].firstChild.nodeValue
-        config['vector_style_file'] = config_dom.getElementsByTagName('VectorStyleFile')[0].firstChild.nodeValue
+        config['vector_layer_contents'] = config_dom.getElementsByTagName('MapfileLayerContents')[0].firstChild.nodeValue
     except IndexError:
         pass
 
@@ -762,6 +763,29 @@ def check_response_code(url, code, code_value=''):
     return False
 
 
+def check_wmts_error(url, code, hash):
+    """
+    Checks WMTS error responses, which often return a HTTP error code and an XML response.
+    Arguments:
+        url (str)-- url to check
+        code (int) -- expected HTTP response code
+        hash (str) -- expected hash value of the response
+    """
+    check_apache_running()
+    try:
+        response = urllib2.urlopen(url)
+        r_code = 200
+    except urllib2.HTTPError as e:
+        r_code = e.code
+        response = e.read()
+    if r_code == code:
+        hasher = hashlib.md5()
+        hasher.update(response)
+        hash_value = str(hasher.hexdigest())
+        return hash_value == hash
+    return False
+
+
 def test_snap_request(hash_table, req_url):
     """
     Requests the first tile for a given layer and date, then compares the result against a dict w/ dates

src/test/test_mod_reproject.py

@@ -122,7 +122,7 @@ def setUpClass(self):
         file_text_replace(os.path.join(self.testdata_path, 'environment_reproject.xml.temp'), os.path.join(self.testdata_path, 'environment_reproject.xml'), '{testfile_dir}', self.testfiles_path)
 
         # Run oe_config_layer to make the cache config files
-        cmd = 'oe_configure_layer --create_mapfile --skip_empty_tiles --generate_links -l{0} -a {1} -c {2} -p {3} -m {4}'.format(self.testfiles_path, self.archive_config, os.path.join(self.testdata_path, 'layer_configuration_file_reproject.xml'), self.projection_config, self.tilematrixset_config)
+        cmd = 'oe_configure_layer --skip_empty_tiles --generate_links -l{0} -a {1} -c {2} -p {3} -m {4}'.format(self.testfiles_path, self.archive_config, os.path.join(self.testdata_path, 'layer_configuration_file_reproject.xml'), self.projection_config, self.tilematrixset_config)
         run_command(cmd)
 
         restart_apache()

src/vectorgen/README.md

@@ -74,4 +74,24 @@ For dense datasets, this option can help improve client performance, as the topm
 
 **email_recipient** - The recipient address for email notifications.
 
-**email_recipient** - The recipient address(es) for email notifications. Use semi-colon ";" to separate recipients.
+**email_recipient** - The recipient address(es) for email notifications. Use semi-colon ";" to separate recipients.
+
+### Feature Filtering (MVT only)
+vectorgen can be configured to pass all the features in a dataset through a set of filters. Features whose metadata passes the filters will be added to the output MVT MRF.
+
+Here is a sample filter block:
+
+```
+<feature_filters>
+    <filter_block logic="OR">
+        <equals name="id" value="some_id"/>
+        <notEquals name="datetime" regexp="^should_not_start_with"/>
+    </filter_block>
+</feature_filters>
+```
+
+**`<feature_filters`>** - This element should appear only once. This contains all the filter data. A feature will be added to the MVT MRF only if it passes **all** the <filter_block> elements.
+
+**`filter_block`** - Defines a single filter set and the logic used to evaluate it. The `logic` attribute is a boolean parameter used to combine all the results of the sub-filters.
+
+**`equals`** and **`notEquals`** - An `equals` test will pass if the metadata property with the given `name` is equal to the given `value` or passes the given `regexp` (if both are present, the regexp is used). A `notEquals` test does the opposite. Regular expression strings must be valid Python regexps.

src/vectorgen/oe_create_mvt_mrf.py

@@ -46,14 +46,14 @@
 import random
 import fiona
 import shapely.geometry
-from rtree import index
+import rtree
 import mapbox_vector_tile
 from osgeo import osr
 import decimal
-        
+import re
 
 # Main tile-creation function.
-def create_vector_mrf(input_file_path, output_path, mrf_prefix, layer_name, target_x, target_y, extents, tile_size, overview_levels, projection_str, feature_reduce_rate=2.5, cluster_reduce_rate=2, debug=False):
+def create_vector_mrf(input_file_path, output_path, mrf_prefix, layer_name, target_x, target_y, extents, tile_size, overview_levels, projection_str, filter_list, feature_reduce_rate=2.5, cluster_reduce_rate=2, debug=False):
     """
     Creates a MVT MRF stack using the specified TileMatrixSet.
 
@@ -71,6 +71,7 @@ def create_vector_mrf(input_file_path, output_path, mrf_prefix, layer_name, targ
         tile_size (int) -- Pixel size of the tiles to be generated.
         overview_levels (list int) -- A list of the overview levels to be used (i.e., a level of 2 will render a level that's 1/2 the width and height of the base level)
         projection_str (str) -- EPSG code for the projection to be used.
+        filter_list (list object) -- List of options for filtering features
         feature_reduce_rate (float) -- (currently only for Point data) Rate at which to reduce features for each successive zoom level.
             Defaults to 2.5 (1 feature retained for every 2.5 in the previous zoom level)
         cluster_reduce_rate (float) -- (currently only for Point data) Rate at which to reduce points in clusters of 1px or less.
@@ -107,7 +108,11 @@ def create_vector_mrf(input_file_path, output_path, mrf_prefix, layer_name, targ
     for input_file in input_file_path:
         print 'Processing ' + input_file
         with fiona.open(input_file) as shapefile:
-            spatial_db = index.Index(rtree_index_generator(list(shapefile)))
+            try:
+                spatial_db = rtree.index.Index(rtree_index_generator(list(shapefile), filter_list))
+            except rtree.core.RTreeError as e:
+                print 'ERROR -- problem importing feature data. If you have filters configured, the source dataset may have no features that pass. Err: {0}'.format(e)
+                sys.exit()
             spatial_dbs.append(spatial_db)
             source_schema = shapefile.schema['geometry']
             source_schemas.append(source_schema)
@@ -362,9 +367,32 @@ def build_mrf_dom(tile_matrices, extents, tile_size, proj):
 # UTILITY STUFF
 
 # This is the recommended way of building an rtree index.
-def rtree_index_generator(features):
+def rtree_index_generator(features, filter_list):
     for idx, feature in enumerate(features):
         try:
-            yield (idx, shapely.geometry.shape(feature['geometry']).bounds, feature)
+            if len(filter_list) == 0 or passes_filters(feature, filter_list):
+                yield (idx, shapely.geometry.shape(feature['geometry']).bounds, feature)            
         except ValueError as e:
             print "WARN - " + str(e)
+
+
+def passes_filters(feature, filter_list):
+    return any(map(lambda filter_block: filter_block_func(feature, filter_block), filter_list))
+
+
+def filter_block_func(feature, filter_block):
+    if filter_block['logic'].lower() == "and":
+        return all(map(lambda comp: filter_func(feature, comp), filter_block['filters']))
+    else:
+        return any(map(lambda comp: filter_func(feature, comp), filter_block['filters']))
+
+
+def filter_func(feature, comparison):
+    property_value = str(feature['properties'].get(comparison['name']))
+    equality = comparison['comparison'] == 'equals'
+    regexp = comparison['regexp']
+    if regexp:
+        result = regexp.search(property_value)
+    else:
+        result = feature['properties'].get(comparison['name']) == comparison['value']
+    return result if equality else not result

src/vectorgen/oe_vectorgen.py

@@ -50,6 +50,7 @@
 import xml.dom.minidom
 import string
 import shutil
+import re
 try:
     from osgeo import ogr, osr, gdal
 except:
@@ -219,6 +220,40 @@ def shp2geojson(in_filename, out_filename, sigevent_url):
                 log_sig_exit('ERROR', "<input_files> or <input_dir> is required", sigevent_url)
             else:
                 input_files = ''
+
+        # Filtering options
+        filter_list = []
+        filter_options = dom.getElementsByTagName('feature_filters')
+        if len(filter_options):
+            for filter_element in filter_options[0].getElementsByTagName('filter_block'):
+                # Validate filter logic
+                logic = filter_element.getAttribute('logic')
+                if not logic:
+                    raise ValueError('"logic" attribute not provided for <filter_block>')
+                if logic.lower() != "and" and logic.lower() != "or":
+                    raise ValueError('Invalid value for "logic" attribute -- must be AND or OR')
+
+                # Get filters
+                comparisons = filter_element.getElementsByTagName('equals') + filter_element.getElementsByTagName('notEquals')
+                def parse_filter(elem):
+                    name = elem.getAttribute('name')
+                    if not name:
+                        raise ValueError('No "name" attribute found for {0} element'.format(elem.nodeName))
+                    value = elem.getAttribute('value')
+                    regexp_str = elem.getAttribute('regexp')
+                    regexp = None
+                    if regexp_str:
+                        try:
+                            regexp = re.compile(regexp_str)
+                        except:
+                            print "ERROR -- problem compiling regexp string {0}. Make sure it's a valid Python regular expression.".format(regexp_str)
+                            sys.exit()
+                    if not value and not regexp:
+                        raise ValueError('No "value" or "regexp" attribute found for {0} element'.format(elem.nodeName))
+                    return {'comparison': elem.nodeName, 'name': name, 'value': value, 'regexp': regexp}
+                filters = list(map(parse_filter, comparisons))
+                filter_list.append({'logic': logic, 'filters': filters})
+
         if output_format == 'mrf':
             log_sig_warn('"MRF" output format not supported, using "MVT-MRF" instead', sigevent_url)
             output_format = 'mvt-mrf'
@@ -385,7 +420,7 @@ def shp2geojson(in_filename, out_filename, sigevent_url):
                     alltiles[idx] = outfile
 
             log_info_mssg("Creating vector mrf with " + ', '.join(alltiles))
-            create_vector_mrf(alltiles, working_dir, basename, tile_layer_name, target_x, target_y, extents, tile_size, overview_levels, target_epsg, feature_reduce_rate=feature_reduce_rate, cluster_reduce_rate=cluster_reduce_rate)
+            create_vector_mrf(alltiles, working_dir, basename, tile_layer_name, target_x, target_y, extents, tile_size, overview_levels, target_epsg, filter_list, feature_reduce_rate=feature_reduce_rate, cluster_reduce_rate=cluster_reduce_rate)
 
             files = [working_dir+"/"+basename+".mrf",working_dir+"/"+basename+".idx",working_dir+"/"+basename+".pvt"]
             for mfile in files:

src/vectorgen/vectorgen_configuration.xsd

@@ -12,32 +12,33 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 -->
-<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema" elementFormDefault="qualified">
+<xs:schema elementFormDefault="qualified" xmlns:xs="http://www.w3.org/2001/XMLSchema">
   <xs:element name="vectorgen_configuration">
     <xs:complexType>
-      <xs:sequence>
+      <xs:all>
         <xs:element ref="date_of_data"/>
-        <xs:element ref="time_of_data" minOccurs="0"/>
+        <xs:element minOccurs="0" ref="time_of_data"/>
         <xs:element ref="parameter_name"/>
-        <xs:element ref="input_files" minOccurs="0"/>
-        <xs:element ref="input_dir" minOccurs="0"/>
+        <xs:element minOccurs="0" ref="input_files"/>
+        <xs:element minOccurs="0" ref="input_dir"/>
         <xs:element ref="output_dir"/>
         <xs:element ref="working_dir"/>
-        <xs:element ref="logfile_dir" minOccurs="0"/>
-        <xs:element ref="output_name" minOccurs="0"/>
+        <xs:element minOccurs="0" ref="logfile_dir"/>
+        <xs:element minOccurs="0" ref="output_name"/>
         <xs:element ref="output_format"/>
-        <xs:element ref="target_epsg" minOccurs="0"/>
-        <xs:element ref="source_epsg" minOccurs="0"/>
+        <xs:element minOccurs="0" ref="target_epsg"/>
+        <xs:element minOccurs="0" ref="source_epsg"/>
         <xs:element ref="target_x"/>
-        <xs:element ref="target_y" minOccurs="0"/>
-        <xs:element ref="extents" minOccurs="0"/>
-        <xs:element ref="tile_size" minOccurs="0"/>
-        <xs:element ref="overview_levels" minOccurs="0"/>
-        <xs:element ref="feature_reduce_rate" minOccurs="0"/>
-        <xs:element ref="cluster_reduce_rate" minOccurs="0"/>
-        <xs:element ref="email_server" minOccurs="0"/>
-        <xs:element ref="email_recipient" minOccurs="0"/>
-      </xs:sequence>
+        <xs:element minOccurs="0" ref="target_y"/>
+        <xs:element minOccurs="0" ref="extents"/>
+        <xs:element minOccurs="0" ref="tile_size"/>
+        <xs:element minOccurs="0" ref="overview_levels"/>
+        <xs:element minOccurs="0" ref="feature_reduce_rate"/>
+        <xs:element minOccurs="0" ref="cluster_reduce_rate"/>
+        <xs:element minOccurs="0" ref="email_server"/>
+        <xs:element minOccurs="0" ref="email_recipient"/>
+        <xs:element minOccurs="0" ref="feature_filters"/>
+      </xs:all>
     </xs:complexType>
   </xs:element>
   <xs:element name="date_of_data" type="xs:integer"/>
@@ -46,15 +47,15 @@ limitations under the License.
   <xs:element name="input_files">
     <xs:complexType>
       <xs:sequence>
-        <xs:element maxOccurs="unbounded" ref="file" minOccurs="0"/>
+        <xs:element maxOccurs="unbounded" minOccurs="0" ref="file"/>
       </xs:sequence>
     </xs:complexType>
   </xs:element>
   <xs:element name="file" type="xs:string"/>
-  <xs:element name="input_dir" type="xs:string" nillable="true"/>
+  <xs:element name="input_dir" nillable="true" type="xs:string"/>
   <xs:element name="output_dir" type="xs:string"/>
   <xs:element name="working_dir" type="xs:string"/>
-  <xs:element name="logfile_dir" type="xs:string" nillable="true"/>
+  <xs:element name="logfile_dir" nillable="true" type="xs:string"/>
   <xs:element name="output_name" type="xs:string"/>
   <xs:element name="output_format">
     <xs:simpleType>
@@ -64,15 +65,57 @@ limitations under the License.
       </xs:restriction>
     </xs:simpleType>
   </xs:element>
-  <xs:element name="target_epsg" type="xs:integer" default="4326"/>
-  <xs:element name="source_epsg" type="xs:integer" default="4326"/>
+  <xs:element default="4326" name="target_epsg" type="xs:integer"/>
+  <xs:element default="4326" name="source_epsg" type="xs:integer"/>
   <xs:element name="target_x" type="xs:integer"/>
-  <xs:element name="target_y" type="xs:integer" nillable="true"/>
-  <xs:element name="extents" type="xs:string" nillable="true"/>
-  <xs:element name="overview_levels" type="xs:string" nillable="true"/>
-  <xs:element name="tile_size" type="xs:integer" nillable="true"/>
-  <xs:element name="feature_reduce_rate" type="xs:float" default="2.5"/>
-  <xs:element name="cluster_reduce_rate" type="xs:float" default="2"/>
-  <xs:element name="email_server" type="xs:string" nillable="true"/>
-  <xs:element name="email_recipient" type="xs:string" nillable="true"/>
-</xs:schema>
+  <xs:element name="target_y" nillable="true" type="xs:integer"/>
+  <xs:element name="extents" nillable="true" type="xs:string"/>
+  <xs:element name="overview_levels" nillable="true" type="xs:string"/>
+  <xs:element name="tile_size" nillable="true" type="xs:integer"/>
+  <xs:element default="2.5" name="feature_reduce_rate" type="xs:float"/>
+  <xs:element default="2" name="cluster_reduce_rate" type="xs:float"/>
+  <xs:element name="email_server" nillable="true" type="xs:string"/>
+  <xs:element name="email_recipient" nillable="true" type="xs:string"/>
+  <xs:element name="feature_filters">
+    <xs:complexType>
+      <xs:sequence>
+        <xs:element name="filter_block">
+          <xs:complexType>
+            <xs:sequence>
+              <xs:element maxOccurs="unbounded" minOccurs="0" name="equals">
+                <xs:complexType>
+                  <xs:simpleContent>
+                    <xs:extension base="xs:string">
+                      <xs:attribute name="name" type="xs:string" use="optional"/>
+                      <xs:attribute name="value" type="xs:string" use="optional"/>
+                      <xs:attribute name="regexp" type="xs:string" use="optional"/>
+                    </xs:extension>
+                  </xs:simpleContent>
+                </xs:complexType>
+              </xs:element>
+              <xs:element maxOccurs="unbounded" minOccurs="0" name="notEquals">
+                <xs:complexType>
+                  <xs:simpleContent>
+                    <xs:extension base="xs:string">
+                      <xs:attribute name="name" type="xs:string" use="optional"/>
+                      <xs:attribute name="regexp" type="xs:string" use="optional"/>
+                      <xs:attribute name="value" type="xs:string" use="optional"/>
+                    </xs:extension>
+                  </xs:simpleContent>
+                </xs:complexType>
+              </xs:element>
+            </xs:sequence>
+            <xs:attribute name="logic" use="required">
+              <xs:simpleType>
+                <xs:restriction base="xs:string">
+                  <xs:enumeration value="OR"/>
+                  <xs:enumeration value="AND"/>
+                </xs:restriction>
+              </xs:simpleType>
+            </xs:attribute>
+          </xs:complexType>
+        </xs:element>
+      </xs:sequence>
+    </xs:complexType>
+  </xs:element>
+</xs:schema>

src/vectorgen/vectorgen_configuration_sample.xml

@@ -16,7 +16,7 @@ limitations under the License.
  <date_of_data>20160429</date_of_data>
  <parameter_name>terra_points_descending</parameter_name>
  <input_files>
-  <file>input_dir/terra_2016-03-04_epsg4326_points_descending.shp</file>
+  <file>test.geojson</file>
  </input_files> 
  <overview_levels>2 4 8</overview_levels>
  <output_dir>output_dir/</output_dir>
@@ -30,4 +30,10 @@ limitations under the License.
  <tile_size>256</tile_size>
  <feature_reduce_rate>0</feature_reduce_rate>
  <cluster_reduce_rate>0</cluster_reduce_rate>
+ <feature_filters>
+    <filter_block logic="AND">
+        <notEquals name="direction" value="Ascending"/>
+        <equals name="direction" regexp="Transitional"/>
+    </filter_block>
+</feature_filters>
 </vectorgen_configuration>