Index records (woudc#1)

Author: tomkralidis

.travis.yml

@@ -5,14 +5,18 @@ python:
 
 env:
   global:
-    - DEBUG=True
-    - DB_TYPE=postgresql
-    - DB_HOST=localhost
-    - DB_PORT=5432
-    - DB_NAME=woudc-data-registry
-    - DB_USERNAME=postgres
-    - DB_PASSWORD=postgres
+    - WDR_DEBUG=True
+    - WDR_DB_TYPE=postgresql
+    - WDR_DB_HOST=localhost
+    - WDR_DB_PORT=5432
+    - WDR_DB_NAME=woudc-data-registry
+    - WDR_DB_USERNAME=postgres
+    - WDR_DB_PASSWORD=postgres
     - PGPASSWORD=postgres
+    - WDR_SEARCH_TYPE=elasticsearch
+    - WDR_SEARCH_URL=http://localhost:9200/
+    - WDR_WAF_BASEURL=https://woudc.org/archive/
+    - WDR_WAF_BASEDIR=/tmp
 
 addons:
   apt:
@@ -35,7 +39,7 @@ script:
   - python setup.py --long-description | rst2html5.py
 
 after_success:
-  - coverage run --source=woudc_data_registry -m unittest woudc_data_registry.tests.run_tests
+  - coverage run --source=woudc_data_registry -m unittest woudc_data_registry.tests.test_data_registry
   - coverage report -m
   - coveralls
   - python setup.py bdist_wheel

Makefile

@@ -53,6 +53,8 @@ help:
 	@echo
 	@echo " createdb: create PostgreSQL/PostGIS database"
 	@echo " dropdb: drop PostgreSQL/PostGIS database"
+	@echo " setup: create models and search index"
+	@echo " teardown: delete models and search index"
 	@echo " test: run tests"
 	@echo " coverage: run code coverage"
 	@echo " package: create Python wheel"
@@ -72,7 +74,7 @@ clean:
 	rm -fr debian/woudc-data-registry
 
 coverage:
-	coverage run --source=woudc_data_registry -m unittest woudc_data_registry.tests.run_tests
+	coverage run --source=woudc_data_registry -m unittest woudc_data_registry.tests.test_data_registry
 	coverage report -m
 
 createdb:
@@ -82,10 +84,21 @@ createdb:
 dropdb:
 	dropdb $(PG_FLAGS)
 
+flake8:
+	flake8 woudc_data_registry
+
 package:
 	python setup.py sdist bdist_wheel
 
+setup:
+	woudc-data-registry manage setup
+	woudc-data-registry search create_index
+
+teardown:
+	woudc-data-registry manage teardown
+	woudc-data-registry search delete_index
+
 test:
 	python setup.py test
 
-.PHONY: clean coverage createdb dropdb help package test
+.PHONY: clean coverage createdb dropdb flake8 help package setup teardown test

README.md

@@ -7,16 +7,17 @@
 
 WOUDC Data Registry is a platform that manages Ozone and Ultraviolet
 Radiation data in support of the [World Ozone and Ultraviolet Radiation Data
-Centre (WOUDC)](http://woudc.org), one of six World Data Centres as part of
+Centre (WOUDC)](https://woudc.org), one of six World Data Centres as part of
 the [Global Atmosphere Watch](http://www.wmo.int/gaw) programme of the
 [WMO](http://www.wmo.int).
 
 
 ## Installation
 
 ### Requirements
-- Python 3 and above
+- [Python](https://www.python.org) 3 and above
 - [virtualenv](https://virtualenv.pypa.io/)
+- [Elasticsearch](https://www.elastic.co/products/elasticsearch) (5.5.0 and above)
 
 ### Dependencies
 Dependencies are listed in [requirements.txt](requirements.txt). Dependencies
@@ -48,13 +49,23 @@ make ENV=foo.env createdb
 make ENV=foo.env dropdb
 
 # initialize model (database tables)
-woudc-data-registry model setup
+woudc-data-registry manage setup
+
+# initialize search engine
+woudc-data-registry search create_index
+
+# load core metadata
+woudc-data-registry manage init
 
 # cleanups
 
 # re-initialize model (database tables)
-woudc-data-registry model teardown
-woudc-data-registry model setup
+woudc-data-registry manage teardown
+woudc-data-registry manage setup
+
+# re-initialize search engine
+woudc-data-registry search delete_index
+woudc-data-registry search create_index
 
 # drop database
 make ENV=foo.env dropdb
@@ -86,13 +97,13 @@ pip install -r requirements-dev.txt
 
 # run tests like this:
 cd woudc_data_registry/tests
-python run_tests.py
+python test_data_registry.py
 
 # or this:
 python setup.py test
 
 # measure code coverage
-coverage run --source=woudc_data_registry -m unittest woudc_data_registry.tests.run_tests
+coverage run --source=woudc_data_registry -m unittest woudc_data_registry.tests.test_data_registry
 coverage report -m
 ```
 

debian/control

@@ -9,8 +9,8 @@ Vcs-Git: https://github.com/woudc/woudc-data-registry.git
 
 Package: woudc-data-registry
 Architecture: all
-Depends: python3-click, python-geoalchemy2, python3-psycopg2, python3-requests, python3-six, python3-sqlalchemy
-Homepage: http://woudc.org
+Depends: elasticsearch (>=5.5.0), postgis, postgresql, python3-click, python-elasticsearch, python-geoalchemy2, python3-psycopg2, python3-requests, python3-six, python3-sqlalchemy
+Homepage: https://woudc.org
 Description: WOUDC Data Registry is a platform that manages Ozone and
  Ultraviolet Radiation data in support of the World Ozone and Ultraviolet
  Radiation Data Centre (WOUDC), one of six World Data Centres as part of the

default.env

@@ -1,8 +1,13 @@
-export DEBUG=False
-export DB_TYPE=postgresql
-export DB_HOST=localhost
-export DB_PORT=5432
-export DB_NAME=woudc-data-registry
-export DB_USERNAME=postgres
-export DB_PASSWORD=postgres
-export PGPASSWORD=$DB_PASSWORD
+export WDR_DEBUG=False
+export WDR_DB_TYPE=postgresql
+export WDR_DB_HOST=localhost
+export WDR_DB_PORT=5432
+export WDR_DB_NAME=woudc-data-registry
+export WDR_DB_USERNAME=postgres
+export WDR_DB_PASSWORD=postgres
+export WDR_SEARCH_TYPE=elasticsearch
+export WDR_SEARCH_URL=http://localhost:9200/
+export WDR_WAF_BASEURL=https://woudc.org/archive/
+export WDR_WAF_BASEDIR=/tmp
+
+export PGPASSWORD=$WDR_DB_PASSWORD

requirements.txt

@@ -1,4 +1,5 @@
 click
+elasticsearch
 geoalchemy2
 psycopg2
 requests

setup.py

@@ -62,7 +62,7 @@ def finalize_options(self):
     def run(self):
         import subprocess
         errno = subprocess.call([sys.executable,
-                                 'woudc_data_registry/tests/run_tests.py'])
+                                 'woudc_data_registry/tests/test_data_registry.py'])  # noqa
         raise SystemExit(errno)
 
 

woudc_data_registry/__init__.py

@@ -45,8 +45,9 @@
 
 import click
 
-from woudc_data_registry.models import model
 from woudc_data_registry.controller import data
+from woudc_data_registry.models import manage
+from woudc_data_registry.search import search
 
 __version__ = '0.1.dev0'
 
@@ -57,5 +58,6 @@ def cli():
     pass
 
 
-cli.add_command(model)
+cli.add_command(manage)
 cli.add_command(data)
+cli.add_command(search)

woudc_data_registry/config.py

@@ -43,22 +43,33 @@
 #
 # =================================================================
 
+import logging
 import os
 
 from woudc_data_registry.util import str2bool
 
+LOGGER = logging.getLogger(__name__)
 
-DEBUG = str2bool(os.getenv('DEBUG', False))
 
-DB_TYPE = os.getenv('DB_TYPE', 'postgresql')
-DB_HOST = os.getenv('DB_HOST', 'localhost')
-DB_PORT = int(os.getenv('DB_PORT', 5432))
-DB_USERNAME = os.getenv('DB_USERNAME', None)
-DB_PASSWORD = os.getenv('DB_PASSWORD', None)
-DB_NAME = os.getenv('DB_NAME', 'woudc-data-registry')
+WDR_DEBUG = str2bool(os.getenv('WDR_DEBUG', False))
 
-if None in [DB_USERNAME, DB_PASSWORD]:
-    raise EnvironmentError('System environment variables are not set!')
+WDR_DB_TYPE = os.getenv('WDR_DB_TYPE', 'postgresql')
+WDR_DB_HOST = os.getenv('WDR_DB_HOST', 'localhost')
+WDR_DB_PORT = int(os.getenv('WDR_DB_PORT', 5432))
+WDR_DB_USERNAME = os.getenv('WDR_DB_USERNAME', None)
+WDR_DB_PASSWORD = os.getenv('WDR_DB_PASSWORD', None)
+WDR_DB_NAME = os.getenv('WDR_DB_NAME', 'woudc-data-registry')
+WDR_SEARCH_TYPE = os.getenv('WDR_SEARCH_TYPE', 'elasticsearch')
+WDR_SEARCH_URL = os.getenv('WDR_SEARCH_URL', 'elasticsearch')
+WDR_WAF_BASEDIR = os.getenv('WDR_WAF_BASEDIR', None)
+WDR_WAF_BASEURL = os.getenv('WDR_WAF_BASEURL', 'https://woudc.org/archive')
 
-DATABASE_URL = '{}://{}:{}@{}:{}/{}'.format(DB_TYPE, DB_USERNAME, DB_PASSWORD,
-                                            DB_HOST, DB_PORT, DB_NAME)
+if None in [WDR_DB_USERNAME, WDR_DB_PASSWORD, WDR_SEARCH_TYPE,
+            WDR_SEARCH_URL, WDR_WAF_BASEDIR, WDR_WAF_BASEURL]:
+    msg = 'System environment variables are not set!'
+    LOGGER.error(msg)
+    raise EnvironmentError(msg)
+
+WDR_DATABASE_URL = '{}://{}:{}@{}:{}/{}'.format(WDR_DB_TYPE, WDR_DB_USERNAME,
+                                                WDR_DB_PASSWORD, WDR_DB_HOST,
+                                                WDR_DB_PORT, WDR_DB_NAME)

woudc_data_registry/controller.py

@@ -50,8 +50,15 @@
 from woudc_data_registry.processing import Process
 
 
-def orchestrate(file_, directory, verify=False):
-    """core workflow"""
+def orchestrate(file_, directory, verify_only=False):
+    """
+    core orchestation workflow
+
+    :param file_: file to process
+    :param directory: directory to process (recursive)
+    :param verify_only: whether to verify the file for correctness without
+                        processing
+    """
 
     files_to_process = []
 
@@ -62,18 +69,19 @@ def orchestrate(file_, directory, verify=False):
             for f in files:
                 files_to_process.append(os.path.join(root, f))
 
-    for file_to_process in files_to_process:
-        click.echo('Processing filename: {}'.format(file_to_process))
-        p = Process()
-        result = p.process_data(file_to_process, verify=verify)
-
-        if result:  # processed
-            if verify:
-                click.echo('Verified but not ingested')
+    with click.progressbar(files_to_process, label='Processing files') as run_:
+        for file_to_process in run_:
+            click.echo('Processing filename: {}'.format(file_to_process))
+            p = Process()
+            result = p.process_data(file_to_process, verify_only=verify_only)
+
+            if result:  # processed
+                if verify_only:
+                    click.echo('Verified but not ingested')
+                else:
+                    click.echo('Ingested successfully')
             else:
-                click.echo('Ingested successfully')
-        else:
-            click.echo('Not ingested')
+                click.echo('Not ingested')
 
 
 @click.group()
@@ -90,8 +98,9 @@ def data():
               type=click.Path(exists=True, resolve_path=True,
                               dir_okay=True, file_okay=False),
               help='Path to directory of data records')
-@click.option('--verify', is_flag=True)
-def ingest(ctx, file_, directory, verify):
+@click.option('--verify-only', '-vo', 'verify_only', is_flag=True,
+              help='Verify file only')
+def ingest(ctx, file_, directory, verify_only):
     """ingest a single data submission or directory of files"""
 
     if file_ is not None and directory is not None:
@@ -102,7 +111,7 @@ def ingest(ctx, file_, directory, verify):
         msg = 'One of --file or --directory is required'
         raise click.ClickException(msg)
 
-    orchestrate(file_, directory, verify)
+    orchestrate(file_, directory, verify_only)
 
 
 data.add_command(ingest)