Merge pull request #5 from todorez/wsgi

convert codi to wsgi, add retries parameters and tests for the RESTful API

Author: bavery22

.travis.yml

@@ -0,0 +1,46 @@
+language: python
+python:
+  - "3.5"
+
+sudo: required
+services:
+  - docker
+
+env:
+  global:
+    - DOCKER_VERSION=1.10.1-0~trusty
+
+before_install:
+  - apt-cache madison docker-engine
+  - sudo apt-get -o Dpkg::Options::="--force-confnew" install -y -qq docker-engine=${DOCKER_VERSION}
+
+script:
+  # configure crops-py dependencies
+  - pip install -e .
+  - pip install --upgrade pip
+  - pip install flask docker-py rethinkdb gunicorn
+
+  # install & configure database
+  - source /etc/lsb-release
+  - echo "deb http://download.rethinkdb.com/apt $DISTRIB_CODENAME main" | sudo tee /etc/apt/sources.list.d/rethinkdb.list
+  - sudo wget -qO- https://download.rethinkdb.com/apt/pubkey.gpg | sudo apt-key add -
+  - sudo apt-get update -qq
+  - sudo apt-get install -y -qq rethinkdb curl
+  - sudo apt-get remove --purge python2.7 -y -qq
+  - sudo cp /etc/rethinkdb/default.conf.sample /etc/rethinkdb/instances.d/default.conf
+  - sudo /etc/init.d/rethinkdb start
+
+  # install toolchain descriptor
+  - sudo mkdir -p /opt/poky/.crops
+  - wget -q -P /tmp http://crops.github.io/toolchain-json/poky-glibc-x86_64-core-image-sato-i586-toolchain-2.1-json.sh
+  - chmod 755 /tmp/poky-glibc-x86_64-core-image-sato-i586-toolchain-2.1-json.sh
+  - sudo /tmp/poky-glibc-x86_64-core-image-sato-i586-toolchain-2.1-json.sh -d /opt/poky -y
+
+  # run tests
+  - python tests.py
+
+notifications:
+  email:
+    recipients:
+    on_success: change
+    on_failure: always

codi/codi.py

@@ -19,6 +19,7 @@
 from flask import Response
 from utils.globs import config
 from utils.docker import dcrops
+from codi import codiDB
 import argparse
 
 class Codi() :
@@ -27,10 +28,10 @@ class Codi() :
     Args: app (Flask): Instance of a Flask application
     '''
 
-    def __init__(self, flask_app, codi_db):
+    def __init__(self, flask_app):
         '''Initialize Codi with a flask app instance variable'''
         self.app = flask_app
-        self.db = codi_db
+        self.db = codiDB.CodiDB(config.CODI_DB)
 
     def list_api(self):
         '''Show CODI API [GET]
@@ -63,13 +64,18 @@ def add_toolchain(self):
         '''
         if request.method == 'POST':
             json_data = request.get_json()
-            #duplicate ids will not be inserted
             json_data["client_ip"] = request.remote_addr
-            self.db.db_insert(config.TOOLCHAINS_TBL, json_data)
-            return 'Success'
+            response = self.db.db_insert(config.TOOLCHAINS_TBL, json_data)
+            if response is not None:
+                response = Response(json.dumps(list(response)),  mimetype='application/json')
+                response.status_code = 200
+            else:
+                response = Response(json.dumps(list(response)),  mimetype='application/json')
+                response.status_code = 400
         else:
-            print("Unable to get JSON data")
-            return 'Error'
+            response = Response(json.dumps(list(response)),  mimetype='application/json')
+            response.status_code = 400
+        return response
 
     def find_image(self):
         '''Search for a toolchain image in Docker repository [GET]
@@ -97,11 +103,9 @@ def pull_image(self):
             cli = dcrops.docker_connect(config.DOCKER_SOCKET)
             image = request.args.get("image")
             if image is not None:
-                for response in cli.pull(image, stream=True):
-                    print(response.decode("utf-8"))
-                    temp = temp + response.decode("utf-8")
+                cli.pull(image, stream=False)
                 cli.close()
-                return Response(json.dumps(temp),  mimetype='application/json')
+                return Response(json.dumps("Success"),  mimetype='application/json')
             else:
                 cli.close()
                 return "Error: Image not provided"
@@ -139,9 +143,9 @@ def get_arg_parser(self):
         returns: codi arguments
         '''
         parser = argparse.ArgumentParser(
-                description='CODI command line arguments')
+            description='CODI command line arguments')
         parser.add_argument('--ip', default="0.0.0.0",
-                help='codi ip address (default: 0.0.0.0)')
+            help='codi ip address (default: 0.0.0.0)')
         parser.add_argument('--port', default=10000, type=int,
-                help='codi port (default: 10000)')
+            help='codi port (default: 10000)')
         return parser.parse_args()

codi/codiDB.py

@@ -15,7 +15,11 @@
 
 from utils.globs import config
 import rethinkdb as r
+import signal
 import json
+import time
+import sys
+import os
 
 class CodiDB() :
     '''CodiDB is used as a persistent storage for tracking
@@ -24,51 +28,83 @@ class CodiDB() :
 
     def __init__(self, db_name):
         self.db_name = db_name
-        self.db_create()
-        self.db_table_create(config.TOOLCHAINS_TBL)
+        create = self.db_create()
+        if create is not None:
+            if self.db_table_create(config.TOOLCHAINS_TBL) is None:
+                print("Unable to initialize database table.Exiting")
+                if ("gunicorn" in os.environ.get("SERVER_SOFTWARE", "")):
+                    os.kill(os.getppid(), signal.SIGTERM)
+                else:
+                    sys.exit(0)
+        else:
+            print("Couldn't connect to the database")
+            if ("gunicorn" in os.environ.get("SERVER_SOFTWARE", "")):
+                os.kill(os.getppid(), signal.SIGTERM)
+            else:
+                sys.exit(0)
 
     def __db_connect_(self):
-        return r.connect('localhost', 28015).repl()
+        try:
+            conn = r.connect('localhost', 28015).repl()
+        except r.ReqlDriverError:
+            return False
+        return conn
+
+    def __db_connect_retry(self, retries):
+        while(bool(retries)):
+            conn =  self.__db_connect_();
+            if (not conn):
+                retries -= 1
+                time.sleep(2)
+            else:
+                return conn
 
     def db_create(self):
-        c = self.__db_connect_()
+        c = self.__db_connect_retry(config.DB_RETRIES)
         res = None
-        if self.db_name not in r.db_list().run(c):
-            res = r.db_create(self.db_name).run(c)
-        c.close()
+        if c :
+            if self.db_name not in r.db_list().run(c):
+                res = r.db_create(self.db_name).run(c)
+            else:
+                res = False
+            c.close()
         return res
 
     def db_table_create(self, table_name):
-        c = self.__db_connect_()
+        c = self.__db_connect_retry(config.DB_RETRIES)
         res = None
-        if table_name not in r.db(self.db_name).table_list().run(c):
-            res =r.db(self.db_name).table_create(table_name).run(c)
-        c.close()
+        if c:
+            if table_name not in r.db(self.db_name).table_list().run(c):
+                res =r.db(self.db_name).table_create(table_name).run(c)
+            else:
+                res = False
+            c.close()
         return res
 
     def db_insert(self, table, entry):
-        c = self.__db_connect_()
-        res = r.db(self.db_name).table(table).insert(entry).run(c)
-        c.close()
+        c = self.__db_connect_retry(config.DB_RETRIES)
+        res = None
+        if c:
+            res = r.db(self.db_name).table(table).insert(entry).run(c)
+            c.close()
         return res
 
     def db_select(self, table, db_filter):
-        c = self.__db_connect_()
-        if db_filter is None:
-            res = r.db(self.db_name).table(table).run(c)
-        else:
-            res = r.db(self.db_name).table(table).filter(json.loads(db_filter)).run(c)
-        c.close()
+        c = self.__db_connect_retry(config.DB_RETRIES)
+        res = None
+        if c:
+            if db_filter is None:
+                res = r.db(self.db_name).table(table).run(c)
+            else:
+                res = r.db(self.db_name).table(table).filter(json.loads(db_filter)).run(c)
+            c.close()
         return res
 
-    def db_update(self):
-        pass
-
     def db_remove(self, table, id):
-        c = self.__db_connect_()
+        c = self.__db_connect_retry(config.DB_RETRIES)
         res = None
-        if id is not None:
-            res = r.db(self.db_name).table(table).get(id).delete().run(c)
-        c.close()
+        if c:
+            if id is not None:
+                res = r.db(self.db_name).table(table).get(id).delete().run(c)
+            c.close()
         return res
-

launchers/codi-launcher.py

@@ -17,6 +17,7 @@
 from codi import codi
 from codi import codiDB
 from utils.globs import config
+import os
 
 def register_codi_routes (app):
     app.add_url_rule('/codi/', 'api_list', codi.list_api)
@@ -30,8 +31,12 @@ def register_codi_routes (app):
 
 if __name__ == '__main__':
     app = Flask(__name__)
-    db = codiDB.CodiDB(config.CODI_DB)
-    codi = codi.Codi(app, db)
-    register_codi_routes(app)
+    codi = codi.Codi(app)
     codi_args = codi.get_arg_parser()
+    register_codi_routes(app)
     app.run(host=codi_args.ip, port=codi_args.port, debug=True)
+
+if ("gunicorn" in os.environ.get("SERVER_SOFTWARE", "")):
+    app = Flask(__name__)
+    codi = codi.Codi(app)
+    register_codi_routes(app)

launchers/gunicorn_config.py

@@ -0,0 +1,22 @@
+#
+# Copyright (C) 2016 Intel Corporation
+#
+# Author: Todor Minchev <[email protected]>
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms and conditions of the GNU General Public License,
+# version 2, or (at your option) any later version, as published by
+# the Free Software Foundation.
+#
+# This program is distributed in the hope it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+
+from codi import codiDB
+
+def db_init(server):
+    codiDB.CodiDB("codi")
+
+on_starting = db_init
+

launchers/turff-launcher.py

@@ -18,11 +18,9 @@
 from utils.globs import config
 import os
 
-if __name__ == '__main__':
+def toolchain_reg(turff):
     response = Response()
-    turff = turff.Turff()
     turff_args = turff.get_arg_parser()
-
     for jfile in os.listdir(turff_args.jsonRoot + "/"):
         if jfile.endswith(".json"):
             jdata = turff.load_json(turff_args.jsonRoot +
@@ -38,3 +36,13 @@
                 print("Registration successful : " + jfile)
             else:
                 print("Registration failed : " + jfile)
+                return False
+    return True
+
+if __name__ == '__main__':
+    turff = turff.Turff()
+    retries = turff.get_arg_parser().retries
+    success = False
+    while (bool(retries) & (not success)):
+        success=toolchain_reg(turff)
+        retries -= 1

tests.py

@@ -0,0 +1,32 @@
+''' crops-py test runner '''
+
+import unittest
+import sys
+
+def run_unittests():
+    ''' Execute Unit Tests '''
+    tests = unittest.TestLoader().discover('tests/unit')
+    result = unittest.TextTestRunner(verbosity=2).run(tests)
+    return result.wasSuccessful()
+
+def run_functional_tests():
+    ''' Execute Functional Tests '''
+    tests = unittest.TestLoader().discover('tests/functional')
+    result = unittest.TextTestRunner(verbosity=2).run(tests)
+    return result.wasSuccessful()
+
+if __name__ == '__main__':
+    print("#" * 70)
+    print("Test Runner: Unit tests")
+    print("#" * 70)
+    unit_results = run_unittests()
+
+    print("#" * 70)
+    print("Test Runner: Functional tests")
+    print("#" * 70)
+    functional_results = run_functional_tests()
+
+    if unit_results and functional_results:
+        sys.exit(0)
+    else:
+        sys.exit(1)