bootstrapping changes

tomkralidis · tomkralidis · commit c039589ef6f8 · 2017-08-01T14:50:19.000Z
diff --git a/.coveragerc b/.coveragerc
@@ -3,3 +3,6 @@ exclude_lines =
     pragma: no cover
     def __repr__
     if __name__ == .__main__.:
+omit =
+    */__init__.py
+    woudc_data_registry/tests/run_tests.py
diff --git a/Makefile b/Makefile
@@ -59,6 +59,10 @@ clean:
 	rm -f debian/woudc-data-registry.substvars
 	rm -fr debian/woudc-data-registry
 
+coverage:
+	coverage run --source=woudc_data_registry -m unittest woudc_data_registry.tests.run_tests
+	coverage report -m
+
 createdb:
 	createdb $(PG_FLAGS) -E UTF8
 	psql $(PG_FLAGS) -c "create extension postgis;"
@@ -69,4 +73,7 @@ dropdb:
 package:
 	python setup.py sdist bdist_wheel
 
-.PHONY: clean createdb dropdb package
+test:
+	python setup.py test
+
+.PHONY: clean coverage createdb dropdb package test
diff --git a/woudc_data_registry/models.py b/woudc_data_registry/models.py
@@ -47,8 +47,8 @@
 import click
 
 import geoalchemy2
-from sqlalchemy import (Column, create_engine, Date, DateTime, Integer, String,
-                        Time, UnicodeText)
+from sqlalchemy import (Boolean, Column, create_engine, Date, DateTime, Enum,
+                        Integer, String, Time, UnicodeText)
 from sqlalchemy.exc import OperationalError, ProgrammingError
 from sqlalchemy.ext.declarative import declarative_base
 
@@ -69,6 +69,42 @@ def get_col_spec(self):
         return '%s(%s,%d)' % (self.name, self.geometry_type, self.srid)
 
 
+class Contributor(base):
+    """Data Registry Contributor"""
+
+    __tablename__ = 'contributor'
+
+    wmo_region_enum = Enum('I', 'II', 'III', 'IV', 'V', 'VI',
+                           name='wmo_region')
+
+    identifier = Column(Integer, primary_key=True, autoincrement=True)
+    acronym = Column(String, nullable=False)
+    name = Column(String, nullable=False)
+    country = Column(String, nullable=False)
+    wmo_region = Column(wmo_region_enum, nullable=False)
+    url = Column(String, nullable=False)
+    email = Column(String, nullable=False)
+    active = Column(Boolean, nullable=False, default=True)
+
+    last_validated_datetime = Column(DateTime, nullable=False,
+                                     default=datetime.utcnow())
+
+    location = Column(Geometry('POINT', srid=4326), nullable=False)
+
+    def __init__(self, dict_):
+        """serializer"""
+
+        self.acronym = dict_['acronym']
+        self.name = dict_['name']
+        self.country = dict_['country']
+        self.wmo_region = dict_['wmo_region']
+        self.url = dict_['url']
+        self.email = dict_['email']
+
+        self.location = util.point2ewkt(dict_['location']['longitude'],
+                                        dict_['location']['latitude'])
+
+
 class DataRecord(base):
     """Data Registry Data Record"""
 
@@ -86,30 +122,40 @@ class DataRecord(base):
     data_generation_date = Column(Date, nullable=False)
     data_generation_agency = Column(String, nullable=False)
     data_generation_version = Column(String, nullable=False)
-    data_generation_scientific_authority = Column(String)
+    data_generation_scientific_authority = Column(String, nullable=True)
 
     platform_type = Column(String, default='STN', nullable=False)
     platform_id = Column(String, nullable=False)
     platform_name = Column(String, nullable=False)
     platform_country = Column(String, nullable=False)
-    platform_gaw_id = Column(String)
+    platform_gaw_id = Column(String, nullable=True)
 
     instrument_name = Column(String, nullable=False)
     instrument_model = Column(String, nullable=False)
     instrument_number = Column(String, nullable=False)
 
-    location = Column(Geometry(srid=0))
+    location = Column(Geometry(srid=0), nullable=False)
 
     timestamp_utcoffset = Column(String, nullable=False)
     timestamp_date = Column(Date, nullable=False)
-    timestamp_time = Column(Time)
+    timestamp_time = Column(Time, nullable=True)
 
     # data management fields
 
-    insert_datetime = Column(DateTime, nullable=False,
-                             default=datetime.utcnow())
+    published = Column(Boolean, nullable=False, default=False)
+
+    received_datetime = Column(DateTime, nullable=False,
+                               default=datetime.utcnow())
+
+    inserted_datetime = Column(DateTime, nullable=False,
+                               default=datetime.utcnow())
+
     processed_datetime = Column(DateTime, nullable=False,
                                 default=datetime.utcnow())
+
+    published_datetime = Column(DateTime, nullable=False,
+                                default=datetime.utcnow())
+
     raw = Column(UnicodeText, nullable=False)
     url = Column(String, nullable=False)
 
@@ -151,6 +197,7 @@ def __init__(self, ecsv):
         self.location = util.point2ewkt(ecsv.extcsv['LOCATION']['Longitude'],
                                         ecsv.extcsv['LOCATION']['Latitude'],
                                         ecsv.extcsv['LOCATION']['Height'])
+
         self.extcsv = ecsv.extcsv
         self.raw = ecsv._raw
 
diff --git a/woudc_data_registry/parser.py b/woudc_data_registry/parser.py
@@ -58,6 +58,29 @@
     'invalid_data': 12000,
 }
 
+DOMAINS = {
+    'datasets': {
+        'Broad-band',
+        'Lidar',
+        'Multi-band',
+        'OzoneSonde',
+        'RocketSonde',
+        'Spectral',
+        'SurfaceOzone',
+        'TotalOzoneObs',
+        'TotalOzone',
+        'UmkehrN14',
+    },
+    'metadata_tables': {
+        'CONTENT': ['Class', 'Category', 'Level', 'Form'],
+        'DATA_GENERATION': ['Date', 'Agency', 'Version'],
+        'PLATFORM': ['Type', 'ID', 'Name', 'Country'],
+        'INSTRUMENT': ['Name', 'Model', 'Number'],
+        'LOCATION': ['Latitude', 'Longitude', 'Height'],
+        'TIMESTAMP': ['UTCOffset', 'Date']
+    }
+}
+
 
 def _get_value_type(field, value):
     """derive true type from data value"""
@@ -101,15 +124,6 @@ def __init__(self, content):
         self.extcsv = {}
         self._raw = None
 
-        self.metadata_tables = {
-            'CONTENT': ['Class', 'Category', 'Level', 'Form'],
-            'DATA_GENERATION': ['Date', 'Agency', 'Version'],
-            'PLATFORM': ['Type', 'ID', 'Name', 'Country'],
-            'INSTRUMENT': ['Name', 'Model', 'Number'],
-            'LOCATION': ['Latitude', 'Longitude', 'Height'],
-            'TIMESTAMP': ['UTCOffset', 'Date']
-        }
-
         LOGGER.debug('Reading into csv')
         self._raw = content
         reader = csv.reader(StringIO(self._raw))
@@ -121,7 +135,7 @@ def __init__(self, content):
         for row in reader:
             if len(row) == 1 and row[0].startswith('#'):  # table name
                     table_name = row[0].replace('#', '')
-                    if table_name in self.metadata_tables.keys():
+                    if table_name in DOMAINS['metadata_tables'].keys():
                         found_table = True
                         LOGGER.debug('Found new table %s', table_name)
                         self.extcsv[table_name] = {}
@@ -136,7 +150,7 @@ def __init__(self, content):
                 LOGGER.debug('Found blank line')
                 continue
             else:  # process row data
-                if table_name in self.metadata_tables.keys():
+                if table_name in DOMAINS['metadata_tables'].keys():
                     self.extcsv[table_name]['_line_num'] = \
                         int(reader.line_num + 1)
                     for idx, val in enumerate(row):
@@ -153,11 +167,11 @@ def validate_metadata(self):
 
         errors = []
 
-        missing_tables = list(set(self.metadata_tables) -
+        missing_tables = list(set(DOMAINS['metadata_tables']) -
                               set(self.extcsv.keys()))
 
         if missing_tables:
-            if not list(set(self.metadata_tables) - set(missing_tables)):
+            if not list(set(DOMAINS['metadata_tables']) - set(missing_tables)):
                 msg = 'No core metadata tables found. Not an Extended CSV file'
                 raise NonStandardDataError(msg)
 
@@ -172,7 +186,7 @@ def validate_metadata(self):
                                           errors)
 
         for key, value in self.extcsv.items():
-            missing_datas = list(set(self.metadata_tables[key]) -
+            missing_datas = list(set(DOMAINS['metadata_tables'][key]) -
                                  set(value.keys()))
 
             if missing_datas:
@@ -205,6 +219,16 @@ def validate_metadata(self):
                     self.extcsv['LOCATION']['_line_num'])
             })
 
+        if self.extcsv['CONTENT']['Category'] not in DOMAINS['datasets']:
+            errors.append({
+                'code': 'invalid_data',
+                'locator': 'CONTENT.Category',
+                'text': 'ERROR: {}: {} (line number: {})'.format(
+                    ERROR_CODES['invalid_data'],
+                    self.extcsv['CONTENT']['Category'],
+                    self.extcsv['LOCATION']['_line_num'])
+            })
+
         if errors:
             raise MetadataValidationError('Invalid metadata', errors)
 
diff --git a/woudc_data_registry/processing.py b/woudc_data_registry/processing.py
@@ -82,6 +82,7 @@ def process_data(self, infile, verify=False):
 
         data_record = None
 
+        LOGGER.info('Detecting file')
         if not is_text_file(infile):
             self.status = 'failed'
             self.code = 'NonStandardDataError'
@@ -118,6 +119,7 @@ def process_data(self, infile, verify=False):
             LOGGER.error('Invalid Extended CSV: {}'.format(err.errors))
             return False
 
+        LOGGER.info('Verifying data record against registry')
         # verify:
         # - Extended CSV core fields against registry
         # - taxonomy/URI check
diff --git a/woudc_data_registry/tests/data/ecsv-invalid-content-category.csv b/woudc_data_registry/tests/data/ecsv-invalid-content-category.csv
@@ -0,0 +1,65 @@
+
+#CONTENT
+Class,Category,Level,Form
+WOUDC,TotalOzoneFOO,1.0,1
+
+#DATA_GENERATION
+Date,Agency,Version,ScientificAuthority
+2012-01-04,RMDA,0.0
+
+#PLATFORM
+Type,ID,Name,Country,GAW_ID
+STN,002,Tamanrasset,DZA
+
+#INSTRUMENT
+Name,Model,Number
+Brewer,MKIII,201
+
+#LOCATION
+Latitude,Longitude,Height
+22.780,95.520,1384
+
+#TIMESTAMP
+UTCOffset,Date,Time
+00:00:00,2011-11-01
+
+#DAILY
+Date,WLCode,ObsCode,ColumnO3,StdDevO3,UTC_Begin,UTC_End,UTC_Mean,NObs,mMu,ColumnSO2
+2011-11-01,9,DS,265.8,2.4,6.37,16.32,11.15,91,1.785,-7.6
+2011-11-02,9,DS,266.6,2.2,6.37,16.20,11.27,99,1.754,-7.9
+2011-11-03,9,DS,273.2,3.4,6.38,16.38,11.47,99,1.782,-7.8
+2011-11-04,9,DS,269.7,2.5,6.53,16.30,11.15,93,1.747,-7.8
+2011-11-05,9,DS,266.4,2.6,6.40,16.28,11.18,86,1.750,-7.6
+2011-11-06,9,DS,262.5,2.8,6.73,15.87,11.29,92,1.746,-7.8
+2011-11-07,9,DS,262.6,2.6,6.57,16.13,11.27,89,1.814,-7.1
+2011-11-08,9,DS,259.2,2.0,6.55,16.25,11.48,97,1.791,-7.2
+2011-11-09,9,DS,260.9,2.9,6.58,16.23,11.38,92,1.823,-7.0
+2011-11-10,9,DS,262.4,3.0,6.45,16.17,11.34,95,1.843,-7.0
+2011-11-11,9,DS,265.6,2.3,6.63,15.77,11.27,91,1.810,-7.1
+2011-11-12,9,DS,268.9,3.9,6.47,16.23,10.92,89,1.805,-7.3
+2011-11-13,9,DS,261.6,2.7,6.65,16.18,12.12,67,2.018,-6.1
+2011-11-14,9,DS,256.7,2.4,7.32,15.95,11.90,80,1.822,-7.5
+2011-11-15,9,DS,269.9,4.4,6.90,16.17,11.71,89,1.807,-7.5
+2011-11-16,9,DS,274.5,3.1,6.65,15.97,11.40,95,1.834,-7.2
+2011-11-17,9,DS,274.6,3.3,7.55,15.37,11.10,67,1.632,-8.5
+2011-11-18,9,DS,270.3,1.6,8.35,15.83,13.01,52,1.728,-7.6
+2011-11-19,9,DS,266.2,2.5,6.68,15.93,11.34,90,1.917,-6.5
+2011-11-20,9,DS,263.1,2.9,6.57,16.12,11.29,90,1.921,-6.7
+2011-11-21,9,DS,263.5,3.4,6.57,16.12,11.53,85,1.892,-7.2
+2011-11-22,9,DS,260.5,2.5,6.58,16.17,11.33,88,1.993,-6.5
+2011-11-23,9,DS,259.4,2.9,6.77,16.05,11.56,90,1.932,-6.9
+2011-11-24,9,DS,256.2,2.4,6.60,15.98,11.47,91,1.938,-6.9
+2011-11-25,9,DS,258.2,2.5,6.82,16.08,11.40,90,1.902,-7.1
+2011-11-26,9,DS,256.8,2.4,6.77,16.17,11.33,81,1.962,-7.0
+2011-11-27,9,DS,254.8,2.3,6.98,15.80,11.03,74,1.921,-7.2
+2011-11-28,9,DS,255.2,3.0,6.97,15.98,11.52,84,1.889,-7.2
+2011-11-29,9,DS,256.3,1.8,6.80,15.77,11.37,66,1.934,-7.1
+2011-11-30,9,DS,262.0,3.1,6.98,15.77,12.52,49,2.103,-5.7
+
+#TIMESTAMP
+UTCOffset,Date,Time
+00:00:00,2011-11-30
+
+#MONTHLY
+Date,ColumnO3,StdDevO3,Npts
+2011-11-01,263.5,5.7,30
diff --git a/woudc_data_registry/tests/data/euc-jp.dat b/woudc_data_registry/tests/data/euc-jp.dat
@@ -0,0 +1 @@
+これはEUC-JPでコーディングされた日本語のファイルです。
diff --git a/woudc_data_registry/tests/run_tests.py b/woudc_data_registry/tests/run_tests.py

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+これはEUC-JPでコーディングされた日本語のファイルです。`