diff --git a/diffy/plugins/diffy_aws/plugin.py b/diffy/plugins/diffy_aws/plugin.py index 36167d5..dc9aeba 100644 --- a/diffy/plugins/diffy_aws/plugin.py +++ b/diffy/plugins/diffy_aws/plugin.py @@ -9,10 +9,11 @@ from typing import List import boto3 -from marshmallow import fields, Schema +from marshmallow import fields from diffy.config import CONFIG from diffy.exceptions import TargetNotFound +from diffy.schema import DiffyInputSchema from diffy.plugins import diffy_aws as aws from diffy.plugins.bases import PersistencePlugin, TargetPlugin, CollectionPlugin @@ -30,7 +31,7 @@ def get_default_aws_account_number() -> dict: return sts.get_caller_identity()['Account'] -class AWSSchema(Schema): +class AWSSchema(DiffyInputSchema): account_number = fields.String(default=get_default_aws_account_number, missing=get_default_aws_account_number) region = fields.String(default=CONFIG['DIFFY_DEFAULT_REGION'], missing=CONFIG['DIFFY_DEFAULT_REGION']) diff --git a/diffy/schema.py b/diffy/schema.py index 2c65239..e738458 100644 --- a/diffy/schema.py +++ b/diffy/schema.py @@ -5,13 +5,89 @@ :license: Apache, see LICENSE for more details. .. moduleauthor:: Kevin Glisson """ -from marshmallow import fields, Schema, post_load + +from inflection import underscore, camelize +from marshmallow import fields, Schema, post_load, pre_load, post_dump from marshmallow.exceptions import ValidationError from diffy.config import CONFIG from diffy.plugins.base import plugins +class DiffySchema(Schema): + """ + Base schema from which all diffy schema's inherit + """ + __envelope__ = True + + def under(self, data, many=None): + items = [] + if many: + for i in data: + items.append( + {underscore(key): value for key, value in i.items()} + ) + return items + return { + underscore(key): value + for key, value in data.items() + } + + def camel(self, data, many=None): + items = [] + if many: + for i in data: + items.append( + {camelize(key, uppercase_first_letter=False): value for key, value in i.items()} + ) + return items + return { + camelize(key, uppercase_first_letter=False): value + for key, value in data.items() + } + + def wrap_with_envelope(self, data, many): + if many: + if 'total' in self.context.keys(): + return dict(total=self.context['total'], items=data) + return data + + +class DiffyInputSchema(DiffySchema): + @pre_load(pass_many=True) + def preprocess(self, data, many): + return self.under(data, many=many) + + +class DiffyOutputSchema(DiffySchema): + @pre_load(pass_many=True) + def preprocess(self, data, many): + if many: + data = self.unwrap_envelope(data, many) + return self.under(data, many=many) + + def unwrap_envelope(self, data, many): + if many: + if data['items']: + self.context['total'] = data['total'] + else: + self.context['total'] = 0 + data = {'items': []} + + return data['items'] + + return data + + @post_dump(pass_many=True) + def post_process(self, data, many): + if data: + data = self.camel(data, many=many) + if self.__envelope__: + return self.wrap_with_envelope(data, many=many) + else: + return data + + def resolve_plugin_slug(slug): """Attempts to resolve plugin to slug.""" plugin = plugins.get(slug) @@ -26,7 +102,7 @@ class PluginOptionSchema(Schema): options = fields.Dict(missing={}) -class PluginSchema(Schema): +class PluginSchema(DiffyInputSchema): options = fields.Dict(missing={}) @post_load @@ -54,3 +130,5 @@ class PayloadPluginSchema(PluginSchema): class AnalysisPluginSchema(PluginSchema): slug = fields.String(missing=CONFIG['DIFFY_ANALYSIS_PLUGIN'], default=CONFIG['DIFFY_ANALYSIS_PLUGIN'], required=True) + + diff --git a/diffy_api/__init__.py b/diffy_api/__init__.py index c9c8445..a5c3988 100644 --- a/diffy_api/__init__.py +++ b/diffy_api/__init__.py @@ -14,11 +14,12 @@ from diffy_api import factory from diffy_api.baseline.views import mod as baseline_bp from diffy_api.analysis.views import mod as analysis_bp - +from diffy_api.tasks.views import mod as task_bp DIFFY_BLUEPRINTS = ( baseline_bp, - analysis_bp + analysis_bp, + task_bp ) diff --git a/diffy_api/analysis/views.py b/diffy_api/analysis/views.py index 1de2200..2a1005b 100644 --- a/diffy_api/analysis/views.py +++ b/diffy_api/analysis/views.py @@ -5,17 +5,17 @@ :license: Apache, see LICENSE for more details. .. moduleauthor:: Kevin Glisson """ -from flask import Blueprint, current_app +from flask import Blueprint, current_app, request from flask_restful import reqparse, Api, Resource -from diffy.core import analysis from diffy.plugins.base import plugins from diffy.exceptions import TargetNotFound -from diffy_api.common.schema import validate_schema +from diffy_api.core import async_analysis +from diffy_api.common.util import validate_schema from diffy_api.schemas import ( analysis_input_schema, - analysis_output_schema, + task_output_schema, ) mod = Blueprint('analysis', __name__) @@ -54,7 +54,7 @@ def get(self): data = plugins.get(current_app.config['DIFFY_PERSISTENCE_PLUGIN']).get_all('analysis') return data, 200 - @validate_schema(analysis_input_schema, None) + @validate_schema(analysis_input_schema, task_output_schema) def post(self, data=None): """ .. http:post:: /analysis @@ -78,7 +78,7 @@ def post(self, data=None): :statuscode 403: unauthenticated """ try: - return analysis(**data) + return async_analysis.queue(**request.json) except TargetNotFound as ex: return {'message': ex.message}, 404 diff --git a/diffy_api/baseline/views.py b/diffy_api/baseline/views.py index 176cd04..7a539e6 100644 --- a/diffy_api/baseline/views.py +++ b/diffy_api/baseline/views.py @@ -5,16 +5,17 @@ :license: Apache, see LICENSE for more details. .. moduleauthor:: Kevin Glisson """ -from flask import Blueprint, current_app +from flask import Blueprint, current_app, request from flask_restful import Api, Resource -from diffy.core import baseline from diffy.plugins.base import plugins from diffy.exceptions import TargetNotFound -from diffy_api.common.schema import validate_schema +from diffy_api.core import async_baseline +from diffy_api.common.util import validate_schema from diffy_api.schemas import ( baseline_input_schema, baseline_output_schema, + task_output_schema, ) @@ -54,7 +55,7 @@ def get(self): data = plugins.get(current_app.config['DIFFY_PERSISTENCE_PLUGIN']).get_all('baseline') return data, 200 - @validate_schema(baseline_input_schema, None) + @validate_schema(baseline_input_schema, task_output_schema) def post(self, data=None): """ .. http:post:: /baselines @@ -78,7 +79,7 @@ def post(self, data=None): :statuscode 403: unauthenticated """ try: - return baseline(**data) + return async_baseline.queue(**request.json) except TargetNotFound as ex: return {'message': ex.message}, 404 diff --git a/diffy_api/common/schema.py b/diffy_api/common/util.py similarity index 50% rename from diffy_api/common/schema.py rename to diffy_api/common/util.py index bbe4191..8297908 100644 --- a/diffy_api/common/schema.py +++ b/diffy_api/common/util.py @@ -1,96 +1,12 @@ -""" -.. module: diffy.common.schema - :platform: unix - :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more - :license: Apache, see LICENSE for more details. - -.. moduleauthor:: Kevin Glisson - -""" -from functools import wraps from typing import List -from flask import request, current_app -from inflection import camelize, underscore -from marshmallow import Schema, post_dump, pre_load +from flask import request, current_app +from functools import wraps +from inflection import camelize from diffy_api.extensions import sentry -class DiffySchema(Schema): - """ - Base schema from which all grouper schema's inherit - """ - __envelope__ = True - - def under(self, data, many=None): - items = [] - if many: - for i in data: - items.append( - {underscore(key): value for key, value in i.items()} - ) - return items - return { - underscore(key): value - for key, value in data.items() - } - - def camel(self, data, many=None): - items = [] - if many: - for i in data: - items.append( - {camelize(key, uppercase_first_letter=False): value for key, value in i.items()} - ) - return items - return { - camelize(key, uppercase_first_letter=False): value - for key, value in data.items() - } - - def wrap_with_envelope(self, data, many): - if many: - if 'total' in self.context.keys(): - return dict(total=self.context['total'], items=data) - return data - - -class DiffyInputSchema(DiffySchema): - @pre_load(pass_many=True) - def preprocess(self, data, many): - return self.under(data, many=many) - - -class DiffyOutputSchema(DiffySchema): - @pre_load(pass_many=True) - def preprocess(self, data, many): - if many: - data = self.unwrap_envelope(data, many) - return self.under(data, many=many) - - def unwrap_envelope(self, data, many): - if many: - if data['items']: - self.context['total'] = data['total'] - else: - self.context['total'] = 0 - data = {'items': []} - - return data['items'] - - return data - - @post_dump(pass_many=True) - def post_process(self, data, many): - if data: - data = self.camel(data, many=many) - if self.__envelope__: - return self.wrap_with_envelope(data, many=many) - else: - return data - - def format_errors(messages: List[str]) -> dict: errors = {} for k, v in messages.items(): @@ -170,4 +86,4 @@ def decorated_function(*args, **kwargs): return unwrap_pagination(resp, output_schema), 200 return decorated_function - return decorator + return decorator \ No newline at end of file diff --git a/diffy_api/core.py b/diffy_api/core.py new file mode 100644 index 0000000..2f26e3f --- /dev/null +++ b/diffy_api/core.py @@ -0,0 +1,19 @@ +from diffy.core import baseline, analysis +from diffy_api.extensions import rq +from diffy_api.schemas import baseline_input_schema, analysis_input_schema + + +@rq.job() +def async_baseline(kwargs): + """Wrapper job around our standard baseline task.""" + # we can't pickle our objects for remote works so we pickle the raw request and then load it here. + data = baseline_input_schema.load(kwargs) + return baseline(**data) + + +@rq.job() +def async_analysis(kwargs): + """Wrapper job around our standard analysis task.""" + # we can't pickle our objects for remote works so we pickle the raw request and then load it here. + data = analysis_input_schema.load(kwargs) + return analysis(**data) diff --git a/diffy_api/extensions.py b/diffy_api/extensions.py index e6a8bcd..f21eec1 100644 --- a/diffy_api/extensions.py +++ b/diffy_api/extensions.py @@ -6,3 +6,6 @@ """ from raven.contrib.flask import Sentry sentry = Sentry() + +from flask_rq2 import RQ +rq = RQ() diff --git a/diffy_api/factory.py b/diffy_api/factory.py index 860bd4c..2da6667 100644 --- a/diffy_api/factory.py +++ b/diffy_api/factory.py @@ -18,7 +18,7 @@ from diffy.common.utils import install_plugins from diffy_api.common.health import mod as health -from diffy_api.extensions import sentry +from diffy_api.extensions import sentry, rq DEFAULT_BLUEPRINTS = ( @@ -75,6 +75,7 @@ def configure_extensions(app): :param app: """ sentry.init_app(app) + rq.init_app(app) def configure_blueprints(app, blueprints): diff --git a/diffy_api/schemas.py b/diffy_api/schemas.py index 4efb435..24ae551 100644 --- a/diffy_api/schemas.py +++ b/diffy_api/schemas.py @@ -12,9 +12,10 @@ PersistencePluginSchema, CollectionPluginSchema, PayloadPluginSchema, - AnalysisPluginSchema + AnalysisPluginSchema, + DiffyInputSchema, + DiffyOutputSchema ) -from diffy_api.common.schema import DiffyInputSchema class BaselineSchema(DiffyInputSchema): @@ -30,8 +31,21 @@ class AnalysisSchema(BaselineSchema): analysis_plugin = fields.Nested(AnalysisPluginSchema, missing={}) +class TaskOutputSchema(DiffyOutputSchema): + id = fields.String(attribute='id') + created_at = fields.DateTime() + arguments = fields.Dict(attribute='kwargs') + status = fields.String(attribute='status') + + +class TaskInputSchema(DiffyInputSchema): + id = fields.String(required=True) + + baseline_input_schema = BaselineSchema() baseline_output_schema = BaselineSchema() analysis_input_schema = AnalysisSchema() analysis_output_schema = AnalysisSchema() - +task_output_schema = TaskOutputSchema() +task_list_output_schema = TaskOutputSchema(many=True) +task_input_schema = TaskInputSchema() \ No newline at end of file diff --git a/diffy_api/tasks/__init__.py b/diffy_api/tasks/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/diffy_api/tasks/views.py b/diffy_api/tasks/views.py new file mode 100644 index 0000000..8ebea46 --- /dev/null +++ b/diffy_api/tasks/views.py @@ -0,0 +1,90 @@ +""" +.. module: diffy.tasks.views + :platform: Unix + :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more + :license: Apache, see LICENSE for more details. +.. moduleauthor:: Kevin Glisson +""" +from flask import Blueprint +from flask_restful import Api, Resource + +from diffy_api.extensions import rq +from diffy_api.common.util import validate_schema +from diffy_api.schemas import ( + task_output_schema, + task_list_output_schema, +) + + +mod = Blueprint('tasks', __name__) +api = Api(mod) + + +class TaskList(Resource): + """Defines the 'taskss' endpoints""" + + def __init__(self): + super(TaskList, self).__init__() + + @validate_schema(None, task_list_output_schema) + def get(self): + """ + .. http:get:: /tasks + The current list of tasks + + **Example request**: + .. sourcecode:: http + GET /tasks HTTP/1.1 + Host: example.com + Accept: application/json, text/javascript + + **Example response**: + .. sourcecode:: http + HTTP/1.1 200 OK + Vary: Accept + Content-Type: text/javascript + + # TODO + + :statuscode 200: no error + :statuscode 403: unauthenticated + """ + queue = rq.get_queue() + return queue.get_jobs() + + +class Task(Resource): + """Defines the 'taskss' endpoints""" + + def __init__(self): + super(Task, self).__init__() + + @validate_schema(None, task_output_schema) + def get(self, task_id): + """ + .. http:get:: /tasks + The current list of tasks + + **Example request**: + .. sourcecode:: http + GET /tasks HTTP/1.1 + Host: example.com + Accept: application/json, text/javascript + + **Example response**: + .. sourcecode:: http + HTTP/1.1 200 OK + Vary: Accept + Content-Type: text/javascript + + # TODO + + :statuscode 200: no error + :statuscode 403: unauthenticated + """ + queue = rq.get_queue() + return queue.fetch_job(task_id) + + +api.add_resource(Task, '/tasks/') +api.add_resource(TaskList, '/tasks', endpoint='tasks') diff --git a/diffy_api/autoapp.py b/diffy_api/wsgi.py similarity index 100% rename from diffy_api/autoapp.py rename to diffy_api/wsgi.py diff --git a/web-requirements.in b/web-requirements.in index 9a8859b..45aaa64 100644 --- a/web-requirements.in +++ b/web-requirements.in @@ -3,4 +3,5 @@ flask raven[flask] flask-restful gunicorn -inflection \ No newline at end of file +inflection +Flask-RQ2 \ No newline at end of file diff --git a/web-requirements.txt b/web-requirements.txt index 4c2c39b..015081b 100644 --- a/web-requirements.txt +++ b/web-requirements.txt @@ -10,11 +10,13 @@ boto3==1.7.9 botocore==1.10.9 click-log==0.2.1 click==6.7 +croniter==0.3.20 # via rq-scheduler deepdiff==3.3.0 docutils==0.14 dogpile.cache==0.6.4 flask-restful==0.3.6 -flask==1.0.1 +flask-rq2==18.0 +flask==1.0.2 fuzzywuzzy==0.16.0 gunicorn==19.8.1 inflection==0.3.1 @@ -32,7 +34,10 @@ python-levenshtein==0.12.0 pytz==2018.4 # via flask-restful pyyaml==3.12 raven[flask]==6.7.0 +redis==2.10.6 # via rq retrying==1.3.3 +rq-scheduler==0.8.2 # via flask-rq2 +rq==0.10.0 # via flask-rq2, rq-scheduler s3transfer==0.1.13 six==1.11.0 swag-client==0.3.8