a lot

fractalis/__init__.py

@@ -5,18 +5,19 @@ Modules in this package:
 """
 from flask import Flask
 
-from fractalis.config import configure_app
 from fractalis.session import RedisSessionInterface
 from fractalis.celery import init_celery
-from fractalis.analytics.controllers import analytics
+from fractalis.analytics.controllers import analytics_blueprint
 
 
-flask_app = Flask(__name__)
-configure_app(flask_app)
+app = Flask(__name__)
+app.config.from_object('fractalis.config')
+app.session_interface = RedisSessionInterface(app.config)
+celery_app = init_celery(app)
 
-flask_app.session_interface = RedisSessionInterface(
-    redis_db_path=flask_app.config['REDIS_DB_PATH'])
+app.register_blueprint(analytics_blueprint, url_prefix='/analytics')
 
-celery_app = init_celery(flask_app)
-
-flask_app.register_blueprint(analytics, url_prefix='/analytics')
+if __name__ == '__main__':
+    app.config.from_envvar('FRACTALIS_CONFIG')
+    celery_app.worker_main(['worker', '--loglevel=DEBUG'])
+    app.run()

fractalis/analytics/controllers.py

@@ -4,20 +4,20 @@ import uuid
 from flask import Blueprint
 
 
-analytics = Blueprint('analytics', __name__)
+analytics_blueprint = Blueprint('analytics_blueprint', __name__)
 
 
-@analytics.route('', methods=['POST'])
+@analytics_blueprint.route('', methods=['POST'])
 def create_job():
     body = json.dumps({'job_id': str(uuid.uuid4())})
     return body, 201
 
 
-@analytics.route('/<uuid:job_id>', methods=['GET'])
+@analytics_blueprint.route('/<uuid:job_id>', methods=['GET'])
 def get_job_details(job_id):
     pass
 
 
-@analytics.route('/<uuid:job_id>', methods=['DELETE'])
+@analytics_blueprint.route('/<uuid:job_id>', methods=['DELETE'])
 def cancel_job(job_id):
     pass

fractalis/analytics/job.py

-from fractalis import celery_app
+"""
 
+"""
 
-def create_job(script, arguments):
-    pass
 
+def get_celery_task(script):
+    split = script.split('.')
+    module = 'fractalis.analytics.scripts.{}'.format(
+        '.'.join(split[:-1]))
+    exec('import {}'.format(module))
+    celery_task = eval('{}.{}'.format(module, split[-1]))
+    return celery_task
 
-def cancel_job(job_id):
-    pass
+
+def start_job(script, arguments):
+    celery_task = get_celery_task(script)
+    async_result = celery_task.delay(**arguments)
+    return async_result.id
 
 
-def get_job_details(job_id):
+def cancel_job(script, job_id):
     pass
 
 
+def get_job_result(script, job_id):
+    celery_task = get_celery_task(script)
+    return celery_task.AsyncResult(job_id)

fractalis/analytics/scripts/test/sample.py

@@ -11,3 +11,8 @@ def add(a, b):
 @celery_app.task
 def do_nothing(time):
     sleep(time)
+
+
+@celery_app.task
+def div(a, b):
+    return a / b

fractalis/celery.py

@@ -27,7 +27,17 @@ def init_celery(app):
     try:
         celery.connection().heartbeat_check()
     except Exception as e:
-        error_msg = "Could not establish connection to {}".format(
+        error_msg = "Could not establish connection to broker: {}".format(
+            app.config['CELERY_BROKER_URL'])
+        raise ConnectionRefusedError(error_msg) from e
+
+    try:
+        @celery.task
+        def f():
+            pass
+        f.delay()
+    except Exception as e:
+        error_msg = "Could not establish connection to backend: {}".format(
             app.config['CELERY_BROKER_URL'])
         raise ConnectionRefusedError(error_msg) from e
 

fractalis/config.py

-"""This module manages the configuration of the Fractalis flask app.
-
-Exports:
-    - configure_app -- Function that configures given Flask app
+""" This file contains the default settings for Fractalis.
 """
-import os
-
-from redislite import StrictRedis
-
-
-class BaseConfig(object):
-    """Basic configuration that should be used in production."""
-    DEBUG = False
-    TESTING = False
-    REDIS_DB_PATH = os.path.join(os.sep, 'tmp', 'fractalis.db')
-    redis = StrictRedis(REDIS_DB_PATH)
-    CELERY_BROKER_URL = 'redis+socket://{}'.format(redis.socket_file)
-    CELERY_RESULT_BACKEND = 'redis+socket://{}'.format(redis.socket_file)
-
-
-class DevelopmentConfig(BaseConfig):
-    """Configuration used in development."""
-    DEBUG = True
-    TESTING = False
-
-
-class TestingConfig(BaseConfig):
-    """Configuration used in testing."""
-    DEBUG = False
-    TESTING = True
-
-
-config = {
-    'development': 'fractalis.config.DevelopmentConfig',
-    'testing': 'fractalis.config.TestingConfig',
-    'production': 'fractalis.config.BaseConfig'
-}
-
-
-def configure_app(app, mode=None):
-    """Apply configuration to given flask app based on environment variable.
-
-    This function assumes that the environment variable FRACTALIS_MODE contains
-    the key 'development', 'testing', 'production', or is unset in which case
-    it defaults to 'production'. Each of these keys maps to a class in this
-    module that contains appropriate settings.
-
-    Keyword Arguments:
-    app (Flask) -- An instance of the app to configure
-    mode (string) -- (optional) Use this instead of the environment variable
-
-    Exceptions:
-    KeyError (Exception) -- Is raised when FRACTALIS_MODE contains unknown key
-    """
-    if mode is None:
-        mode = os.getenv('FRACTALIS_MODE', default='production')
 
-    try:
-        app.config.from_object(config[mode])
-    except KeyError as e:
-        raise KeyError("'{}' is no valid value for the FRACTALIS_MODE "
-                       "environment variable.".format(mode)) from e
+# DO NOT MODIFY THIS FILE!
+DEBUG = False
+TESTING = False
+REDIS_HOSTNAME = '127.0.0.1'
+REDIS_PORT = '6379'
+CELERY_BROKER_URL = 'amqp://'
+CELERY_RESULT_BACKEND = 'redis://127.0.0.1:6379'
+# DO NOT MODIFY THIS FILE!

fractalis/session.py

+import math
+import datetime
 from uuid import uuid4
 
-from redislite import StrictRedis
+from redis import StrictRedis
 from flask.sessions import SecureCookieSessionInterface, SecureCookieSession
 
 
 class RedisSession(SecureCookieSession):
+    """An implementation of SecureCookieSession that expands the class with
+    a sid field."""
 
     def __init__(self, sid, initial=None):
         super().__init__(initial=initial)
@@ -12,9 +16,17 @@ class RedisSession(SecureCookieSession):
 
 
 class RedisSessionInterface(SecureCookieSessionInterface):
+    """An implementation of SecureCookieSessionInterface that makes use of
+    Redis as a session storage.
 
-    def __init__(self, redis_db_path):
-        self.redis = StrictRedis(redis_db_path)
+    Fields:
+    redis (StrictRedis) -- The connection to the Redis database
+    sid (UUID) -- A session id
+    """
+
+    def __init__(self, app_config):
+        self.redis = StrictRedis(host=app_config['REDIS_HOSTNAME'],
+                                 port=app_config['REDIS_PORT'])
 
     def open_session(self, app, request):
         sid = request.cookies.get(app.session_cookie_name)
@@ -34,10 +46,38 @@ class RedisSessionInterface(SecureCookieSessionInterface):
             if session.modified:
                 response.delete_cookie(app.session_cookie_name, domain=domain)
             return None
-        session_expiration_time = self.get_expiration_time(app, session)
+        expiration_times = self.get_expiration_times(app, session)
         serialzed_session_data = self.serializer.dumps(dict(session))
-        self.redis.setex('session:{}'.format(session.sid),
-                         session_expiration_time, serialzed_session_data)
-        response.set_cookie(app.session_cookie_name, session.sid,
-                            expires=session_expiration_time, httponly=True,
+        self.redis.setex(name='session:{}'.format(session.sid),
+                         time=expiration_times['redis'],
+                         value=serialzed_session_data)
+        response.set_cookie(key=app.session_cookie_name, value=session.sid,
+                            expires=expiration_times['cookies'], httponly=True,
                             domain=domain)
+
+    def get_expiration_times(self, app, session):
+        """Get dictionary that contains redis session and cookie expiration
+        times in the correct format.
+
+        We need this method for two reasons. First, if the expiration time is
+        None we need to set it to a default. Second, there is a bug that
+        prohibits redislite.Redis.setex method to use a datetime object for
+        expiration time, so this method converts it to integer (seconds).
+
+        Keyword Arguments:
+        app (Flask) -- An instance of a Flask application
+        session (SecureCookieSession) -- An instance of a session
+
+        Returns:
+        (dict) -- A dict containing expiration times for redis and cookie
+        """
+        expiration_times = {'redis': 60 * 60 * 24, 'cookies': None}
+        now = datetime.datetime.utcnow()
+        session_expiration_time = self.get_expiration_time(app, session)
+        if session_expiration_time is not None:
+            seconds = (session_expiration_time - now).total_seconds()
+            expiration_times['redis'] = math.ceil(seconds)
+        cookie_expiration_time = (now + datetime.timedelta(
+            seconds=expiration_times['redis']))
+        expiration_times['cookies'] = cookie_expiration_time
+        return expiration_times

run.py

-from fractalis import app
-
-if __name__ == '__main__':
-    app.run()

setup.py

@@ -8,8 +8,8 @@ setup(
     packages=find_packages(),
     install_requires=[
         'Flask',
-        'celery',
-        'redislite'
+        'celery[redis]',
+        'redis'
     ],
     setup_requires=[
         'pytest-runner',

tests/test_analytics.py

 import uuid
-import json
 
+import flask
 import pytest
 
 
 class TestAnalytics(object):
 
-    @pytest.fixture
-    def flask_app(self):
+    @pytest.fixture(scope='module')
+    def app(self):
         from flask import Flask
-        from fractalis.config import configure_app
         app = Flask('test_app')
-        configure_app(app, mode='testing')
+        app.testing = True
         test_client = app.test_client()
         return test_client
 
-    def test_new_resource_created(self, flask_app):
-        response = flask_app.post('/analytics', data=dict(
+    def test_new_resource_created(self, app):
+        response = app.post('/analytics', data=dict(
             script='test/sample.py',
             arguments={'a': 1, 'b': 1}
         ))
-        response_body = json.loads(response.get_data().decode('utf-8'))
+        response_body = flask.json.loads(response.get_data())
         new_resource_url = '/analytics/{}'.format(response_body['job_id'])
         assert response.status_code == 201
         assert uuid.UUID(response_body['job_id'])
-        assert flask_app.head(new_resource_url).status_code == 200
+        assert app.head(new_resource_url).status_code == 200
 
-    def test_400_if_creating_but_script_does_not_exist(self, flask_app):
-        response = flask_app.post('/analytics', data=dict(
+    def test_400_if_creating_but_script_does_not_exist(self, app):
+        response = app.post('/analytics', data=dict(
             script='test/sapmle.py',
             arguments={'a': 1, 'b': 1}
         ))
-        response_body = json.loads(response.get_data().decode('utf-8'))
+        response_body = flask.json.loads(response.get_data())
         assert response.status_code == 400
         assert response_body['error_msg']
 
-    def test_400_if_creating_but_arguments_are_invalid(self, flask_app):
-        response = flask_app.post('/analytics', data=dict(
+    def test_400_if_creating_but_arguments_are_invalid(self, app):
+        response = app.post('/analytics', data=dict(
             script='test/sample.py',
             arguments={'a': 1, 'c': 1}
         ))
-        response_body = json.loads(response.get_data().decode('utf-8'))
+        response_body = flask.json.loads(response.get_data())
         assert response.status_code == 400
         assert response_body['error_msg']
 
-    def test_403_if_creating_but_not_authenticated(self, flask_app):
-        response = flask_app.post('/analytics', data=dict(
+    def test_403_if_creating_but_not_authenticated(self, app):
+        response = app.post('/analytics', data=dict(
             script='test/sample.py',
             arguments={'a': 1, 'b': 1}
         ))
-        response_body = json.loads(response.get_data().decode('utf-8'))
         assert response.status_code == 403
 
-    def test_resource_deleted(self, flask_app):
-        response = flask_app.post('/analytics')
-        response_body = json.loads(response.get_data().decode('utf-8'))
+    def test_resource_deleted(self, app):
+        response = app.post('/analytics')
+        response_body = flask.json.loads(response.get_data())
         new_resource_url = '/analytics/{}'.format(response_body['job_id'])
-        assert flask_app.delete(new_resource_url).status_code == 200
-        assert flask_app.head(new_resource_url).status_code == 404
+        assert app.delete(new_resource_url).status_code == 200
+        assert app.head(new_resource_url).status_code == 404
 
-    def test_403_if_deleting_but_not_authenticated(self, flask_app):
+    def test_403_if_deleting_but_not_authenticated(self, app):
         assert False
 
-    def test_404_if_deleting_non_existing_resource(self, flask_app):
+    def test_404_if_deleting_non_existing_resource(self, app):
         assert False
 
-    def test_403_when_getting_status_but_not_authenticated(self, flask_app):
+    def test_403_when_getting_status_but_not_authenticated(self, app):
         assert False
 
-    def test_status_result_non_empty_if_finished(self, flask_app):
+    def test_status_result_non_empty_if_finished(self, app):
         assert False
 
-    def test_status_result_empty_if_not_finished(self, flask_app):
+    def test_status_result_empty_if_not_finished(self, app):
         assert False
 
-    def test_404_if_status_non_existing_resource(self, flask_app):
+    def test_404_if_status_non_existing_resource(self, app):
         assert False

tests/test_celery.py

@@ -6,21 +6,20 @@ from fractalis.celery import init_celery
 
 class TestCelery(object):
 
-    @pytest.fixture
+    @pytest.fixture()
     def app(self):
         from flask import Flask
-        from fractalis.config import configure_app
         app = Flask('test_app')
-        configure_app(app, mode='testing')
+        app.config.from_object('fractalis.config')
         return app
 
     def test_exception_if_no_connection_to_broker(self, app):
-        app.config['CELERY_BROKER_URL'] = 'redis+socket:///foobar.socket'
-        with pytest.raises(ConnectionError):
+        app.config['CELERY_BROKER_URL'] = 'redis://lacolhost:6379'
+        with pytest.raises(ConnectionRefusedError):
             init_celery(app)
 
     def test_exception_if_no_connection_to_result_backend(self, app):
-        app.config['CELERY_RESULT_BACKEND'] = 'redis+socket:///foobar.socket'
+        app.config['CELERY_RESULT_BACKEND'] = 'redis://lacolhost:6379'
         with pytest.raises(ConnectionRefusedError):
             init_celery(app)
 

tests/test_config.py

-import os
-from importlib import reload
-
-import pytest
-
-import fractalis
-
-
-class TestConfig(object):
-
-    def test_config_when_test_mode(self):
-        os.environ['FRACTALIS_MODE'] = 'testing'
-        reload(fractalis)
-        assert not fractalis.flask_app.config['DEBUG']
-        assert fractalis.flask_app.config['TESTING']
-
-    def test_config_when_development_mode(self):
-        os.environ['FRACTALIS_MODE'] = 'development'
-        reload(fractalis)
-        assert fractalis.flask_app.config['DEBUG']
-        assert not fractalis.flask_app.config['TESTING']
-
-    def test_config_when_production_mode(self):
-        os.environ['FRACTALIS_MODE'] = 'production'
-        reload(fractalis)
-        assert not fractalis.flask_app.config['DEBUG']
-        assert not fractalis.flask_app.config['TESTING']
-
-    def test_config_when_default(self):
-        del os.environ['FRACTALIS_MODE']
-        reload(fractalis)
-        assert not fractalis.flask_app.config['DEBUG']
-        assert not fractalis.flask_app.config['TESTING']
-
-    def test_config_when_unknown_mode(self):
-        os.environ['FRACTALIS_MODE'] = 'foobar'
-        with pytest.raises(KeyError):
-            reload(fractalis)

tests/test_job.py

@@ -10,40 +10,47 @@ class TestJob(object):
 
     def test_exception_when_starting_non_existing_script(self):
         with pytest.raises(ImportError):
-            job.create_job('querty', {})
+            job.start_job('querty.sample.add', {})
 
     def test_exception_when_invalid_parameters(self):
         with pytest.raises(TypeError):
-            job.create_job('test/sample/add', {'a': 1})
+            job.start_job('test.sample.add', {'a': 1})
 
     def test_start_job_returns_uuid(self):
-        job_id = job.create_job('test/sample/add', {})
+        job_id = job.start_job('test.sample.add', {'a': 1, 'b': 2})
         UUID(job_id)
 
-    def test_job_in_progress_has_running_status(self):
-        job_id = job.create_job('test/sample/do_nothing', {'time': 2})
-        job_details = job.get_job_details(job_id)
-        assert job_details.status == 'RUNNING'
-
     def test_finished_job_returns_results(self):
-        job_id = job.create_job('test/sample/add', {'a': 1, 'b': 2})
+        job_id = job.start_job('test.sample.add', {'a': 1, 'b': 2})
+        sleep(1)
+        async_result = job.get_job_result('test.sample.add', job_id)
+        assert async_result.status == 'SUCCESS'
+        assert async_result.result == 3
+
+    def test_failing_job_return_exception_message(self):
+        job_id = job.start_job('test.sample.div', {'a': 1, 'b': 0})
         sleep(1)
-        job_details = job.get_job_details(job_id)
-        assert job_details.status == 'FINISHED'
-        assert job_details.message == 3
+        async_result = job.get_job_result('test.sample.div', job_id)
+        assert async_result.status == 'FAILURE'
+        assert async_result.result == 'wdawd'
+
+    def test_job_in_progress_has_running_status(self):
+        job_id = job.start_job('test.sample.do_nothing', {'time': 2})
+        async_result = job.get_job_result('test.sample.do_nothing', job_id)
+        assert async_result.status == 'PENDING'
 
     def test_exception_when_checking_non_existing_job(self):
         with pytest.raises(LookupError):
-            job.get_job_details(uuid4())
+            job.get_job_result('test.sample.do_nothing', str(uuid4()))
 
     def test_job_is_gone_after_canceling(self):
-        job_id = job.create_job('test/sample/do_nothing', {'time': 10})
-        job.cancel_job(job_id)
+        job_id = job.start_job('test.sample.do_nothing', {'time': 10})
+        job.cancel_job('test.sample.do_nothing', job_id)
         # TODO Not sure which exception is thrown
         with pytest.raises():
-            job.get_job_details(job_id)
+            job.get_job_result(job_id)
 
     def test_exception_when_canceling_non_existing_job(self):
         # TODO Not sure which exception is thrown
         with pytest.raises():
-            job.cancel_job(uuid4())
+            job.cancel_job('test.sample.do_nothing', uuid4())

tests/test_session.py

+from uuid import UUID
 from time import sleep
 
 import pytest
 import flask
-from redislite import StrictRedis
+from redis import StrictRedis
 
 
 class TestSession(object):
 
-    @pytest.fixture
-    def flask_app(self):
+    @pytest.fixture(scope='module')
+    def app(self):
         from flask import Flask
-        from fractalis import configure_app
         from fractalis.session import RedisSessionInterface
         app = Flask('test_app')
-        configure_app(app, mode='testing')
-        app.session_interface = RedisSessionInterface(
-            redis_db_path=app.config['REDIS_DB_PATH'])
+        app.config.from_object('fractalis.config')
+        app.testing = True
+        app.session_interface = RedisSessionInterface(app.config)
         return app
 
-    def test_add_data_to_session_and_expect_it_in_db(self, flask_app):
-        redis = StrictRedis(flask_app.config['REDIS_DB_PATH'])
-        with flask_app.test_client() as test_client:
-            with test_client.session_transaction() as session:
-                session['foo'] = 'bar'
-            session_id = flask.session.sid
-            assert redis.get('session:{}'.format(session_id))['foo'] == 'bar'
+    @pytest.fixture(scope='module')
+    def redis(self, app):
+        redis = StrictRedis(host=app.config['REDIS_HOSTNAME'],
+                            port=app.config['REDIS_PORT'])
+        return redis
 
-    def test_add_data_and_expect_cookie_set(self, flask_app):
-        with flask_app.test_client() as test_client:
-            with test_client.session_transaction() as session:
-                session['foo'] = 'bar'
-            test_client.get()
+    def test_add_data_to_session_and_expect_it_in_db(self, app, redis):
+        with app.test_client() as c:
+            with c.session_transaction() as sess:
+                sess.permanent = True
+                sess['foo'] = 'bar'
+                session_id = sess.sid
+        value = redis.get('session:{}'.format(session_id))
+        assert flask.json.loads(value)['foo'] == 'bar'
+
+    def test_add_data_to_session_and_expect_sid_to_be_uuid(self, app):
+        with app.test_client() as c:
+            with c.session_transaction() as sess:
+                sess.permanent = True
+                sess['foo'] = 'bar'
+                assert sess.sid
+                UUID(sess.sid)
+
+    def test_add_data_and_expect_cookie_set(self, app):
+        with app.test_client() as c:
+            with c.session_transaction() as sess:
+                sess.permanent = True
+                sess['foo'] = 'bar'
+            c.get()
             assert flask.request.cookies
 
-    def test_dont_add_data_and_exoect_no_cookie_set(self, flask_app):
-        with flask_app.test_client() as test_client:
-            test_client.get()
+    def test_dont_add_data_and_exoect_no_cookie_set(self, app):
+        with app.test_client() as c:
+            c.get()
             assert not flask.request.cookies
 
-    def test_change_session_data_and_expect_change_in_db(self, flask_app):
-        redis = StrictRedis(flask_app.config['REDIS_DB_PATH'])
-        with flask_app.test_client() as test_client:
-            with test_client.session_transaction() as session:
-                session['foo'] = 'bar'
-            session_id = flask.session.sid
-            assert redis.get('session:{}'.format(session_id))['foo'] == 'bar'
-            with test_client.session_transaction() as session:
-                session['foo'] = 'baz'
-            assert redis.get('session:{}'.format(session_id))['foo'] == 'baz'
+    def test_change_session_data_and_expect_change_in_db(self, app, redis):
+        with app.test_client() as c:
+            with c.session_transaction() as sess:
+                sess.permanent = True
+                sess['foo'] = 'bar'
+            session_id = sess.sid
+        value = redis.get('session:{}'.format(session_id))
+        assert flask.json.loads(value)['foo'] == 'bar'
+        with app.test_client() as c:
+            with c.session_transaction() as sess:
+                sess.permanent = True
+                sess['foo'] = 'baz'
+            session_id = sess.sid
+        value = redis.get('session:{}'.format(session_id))
+        assert flask.json.loads(value)['foo'] == 'baz'
+
+    def test_session_data_not_in_db_when_expired(self, app, redis):
+        app.config['PERMANENT_SESSION_LIFETIME'] = 1
+        with app.test_client() as c:
+            with c.session_transaction() as sess:
+                sess.permanent = True
+                sess['foo'] = 'bar'
+                session_id = sess.sid
+        sleep(2)
+        assert not redis.get('session:{}'.format(session_id))
 
-    def test_exception_when_manipulating_session_data(self, flask_app):
+    def test_exception_when_manipulating_session_data(self, app):
         # No need to test this atm because we store nothing in the cookie
         assert True
-
-    def test_exception_when_accessing_expired_session_data(self, flask_app):
-        flask_app.config['PERMANENT_SESSION_LIFETIME'] = 1
-        with flask_app.test_client() as test_client:
-            with test_client.session_transaction() as session:
-                session['foo'] = 'bar'
-            sleep(2)
-            #TODO Not sure which exception is thrown
-            flask.session['foo']
-
-    def test_session_data_not_in_db_when_expired(self, flask_app):
-        flask_app.config['PERMANENT_SESSION_LIFETIME'] = 1
-        redis = StrictRedis(flask_app.config['REDIS_DB_PATH'])
-        with flask_app.test_client() as test_client:
-            with test_client.session_transaction() as session:
-                session['foo'] = 'bar'
-            sleep(2)
-            session_id = flask.session.sid
-            assert not redis.get('session:{}'.format(session_id))['foo']