update countries model, add doc, add exceptions for search

Author: tomkralidis

Makefile

@@ -54,6 +54,7 @@ help:
 	@echo " createdb: create PostgreSQL/PostGIS database"
 	@echo " dropdb: drop PostgreSQL/PostGIS database"
 	@echo " setup: create models and search index"
+	@echo " setup_data: download core metadata
 	@echo " teardown: delete models and search index"
 	@echo " test: run tests"
 	@echo " coverage: run code coverage"
@@ -72,6 +73,7 @@ clean:
 	rm -f debian/woudc-data-registry.prerm.debhelper
 	rm -f debian/woudc-data-registry.substvars
 	rm -fr debian/woudc-data-registry
+	@echo "NOT removing data/"
 
 coverage:
 	coverage run --source=woudc_data_registry -m unittest woudc_data_registry.tests.test_data_registry
@@ -94,11 +96,15 @@ setup:
 	woudc-data-registry manage setup
 	woudc-data-registry search create-index
 
+setup_data:
+	mkdir -p data
+	curl -o data/wmo-countries.json https://www.wmo.int/cpdb/data/membersandterritories.json
+
 teardown:
 	woudc-data-registry manage teardown
 	woudc-data-registry search delete-index
 
 test:
 	python setup.py test
 
-.PHONY: clean coverage createdb dropdb flake8 help package setup teardown test
+.PHONY: clean coverage createdb dropdb flake8 help package setup setup_data teardown test

README.md

@@ -55,7 +55,11 @@ woudc-data-registry manage setup
 woudc-data-registry search create-index
 
 # load core metadata
-woudc-data-registry manage init
+
+# fetch WMO country list
+mkdir data
+curl -o data/wmo-countries.json https://www.wmo.int/cpdb/data/membersandterritories.json
+woudc-data-registry manage init -d data/
 
 # cleanups
 

woudc_data_registry/models.py

@@ -63,7 +63,8 @@
 
 LOGGER = logging.getLogger(__name__)
 
-WMO_REGION_ENUM = Enum('I', 'II', 'III', 'IV', 'V', 'VI', name='wmo_region')
+WMO_REGION_ENUM = Enum('I', 'II', 'III', 'IV', 'V', 'VI', 'Antarctica',
+                       name='wmo_region')
 
 
 class Geometry(geoalchemy2.types.Geometry):
@@ -79,17 +80,22 @@ def get_col_spec(self):
 
 
 class Country(base):
-    """Data Registry Country"""
+    """
+    Data Registry Country
+
+    https://www.wmo.int/cpdb/data/membersandterritories.json
+
+    """
 
     __tablename__ = 'countries'
 
     identifier = Column(String, nullable=False, primary_key=True)
     country_name = Column(String, nullable=False, unique=True)
     french_name = Column(String, nullable=False, unique=True)
-    wmo_region_id = Column(String, nullable=False)
-    regional_involvement = Column(String, nullable=False)
-    wmo_membership = Column(Date, nullable=True)
-    link = Column(String, nullable=True)
+    wmo_region_id = Column(WMO_REGION_ENUM, nullable=False)
+    regional_involvement = Column(WMO_REGION_ENUM, nullable=False)
+    wmo_membership = Column(Date, nullable=False)
+    link = Column(String, nullable=False)
 
     def __init__(self, dict_):
         self.identifier = dict_['id']
@@ -101,7 +107,7 @@ def __init__(self, dict_):
         self.link = dict_['link']
 
     def __repr__(self):
-        return 'Country ({}, {})'.format(self.identifier, self.name)
+        return 'Country ({}, {})'.format(self.identifier, self.country_name)
 
 
 class Contributor(base):
@@ -175,7 +181,7 @@ class Station(base):
 
     stn_type_enum = Enum('STN', 'SHP', name='type')
 
-    identifier = Column(Integer, primary_key=True)
+    identifier = Column(String, primary_key=True)
     name = Column(String, nullable=False)
     contributor_id = Column(String, ForeignKey('contributors.identifier'),
                             nullable=True)
@@ -201,6 +207,7 @@ class Station(base):
     def __init__(self, dict_):
         """serializer"""
 
+        self.identifier = dict_['identifier']
         self.name = dict_['name']
         self.stn_type = dict_['stn_type']
         self.gaw_id = dict_['gaw_id']
@@ -212,7 +219,7 @@ def __init__(self, dict_):
         self.location = util.point2ewkt(dict_['x'], dict_['y'], dict_['z'])
 
     def __repr__(self):
-        return 'Station ({}, {})'.format(self.stn_identifier, self.name)
+        return 'Station ({}, {})'.format(self.identifier, self.name)
 
 
 class DataRecord(base):
@@ -440,7 +447,7 @@ def init(ctx, datadir):
     if datadir is None:
         raise click.ClickException('Missing required data directory')
 
-    countries = os.path.join(datadir, 'countries.json')
+    countries = os.path.join(datadir, 'wmo-countries.json')
     contributors = os.path.join(datadir, 'contributors.csv')
     stations = os.path.join(datadir, 'stations.csv')
     datasets = os.path.join(datadir, 'datasets.csv')
@@ -456,6 +463,19 @@ def init(ctx, datadir):
                 continue
             country = Country(country_data)
             registry_.save(country)
+    # Antarctica is not a recognized country per se but is
+    # provided in WOUDC
+    antarctica = {
+        'id': 'ATA',
+        'country_name': 'Antarctica',
+        'french_name': 'Antarctique',
+        'wmo_region_id': 'Antarctica',
+        'regional_involvement': 'Antarctica',
+        'wmo_membership': '1970-01-01',
+        'link': 'https://www.wmo.int/pages/prog/www/Antarctica/Purpose.html'
+    }
+    country = Country(antarctica)
+    registry_.save(country)
 
     click.echo('Loading datasets metadata')
     with open(datasets) as csvfile:

woudc_data_registry/search.py

@@ -49,7 +49,8 @@
 
 import click
 from elasticsearch import Elasticsearch
-from elasticsearch.exceptions import NotFoundError, RequestError
+from elasticsearch.exceptions import (ConnectionError, NotFoundError,
+                                      RequestError)
 import requests
 
 from woudc_data_registry import config
@@ -94,7 +95,7 @@ def create(self):
         try:
             self.connection.indices.create(index=self.index_name,
                                            body=settings)
-        except RequestError as err:
+        except (ConnectionError, RequestError) as err:
             LOGGER.error(err)
             raise SearchIndexError(err)