Overwriting session behaviour to use Redis

doc/api/analytics/root.yml

-/analytics:
-  post:
-    summary: Launch analytical script
-    parameters:
-      - name: script
-        in: query
-        description: Analysis script to run
-        required: true
-        type: string
-      - name: arguments
-        in: query
-        description: String describing the arguments to the analysis script
-        required: true
-        type: string
-    responses:
-      201:
-        description: Created
-        schema:
-          type: object
-          properties:
-            job_id:
-              type: string

doc/api/swagger.yaml

+swagger: '2.0'
+info:
+  title: Fractalis API
+  description: foobar
+  version: '0.0.0'
+host: 127.0.0.1:5000
+schemes:
+  - http
+produces:
+  - application/json
+paths:
+  /data:
+    post:
+      summary: Add data to current session
+      parameters:
+        - name: etl
+          in: query
+          description: ETL to use for moving data from source to Fractalis
+          required: false
+          type: string
+        - name: data
+          in: query
+          description: Data in Fractalis standard format
+          required: false
+          type: string
+        - name: action
+          in: query
+          description: Action to extract the data from the source system
+          required: false
+          type: string
+      responses:
+        201:
+          description: Accepted
+          schema:
+            type: object
+            properties:
+              data_id:
+                type: string
+    get:
+      summary: Get all completed and running ETLs for current session
+      responses:
+        200:
+          description: OK
+          schema:
+            type: array
+            items:
+              $ref: '#/definitions/ETLStatus'
+  /data/{data_id}:
+    parameters:
+      - name: data_id
+        in: path
+        description: ID given on launching an ETL
+        required: true
+        type: string
+    get:
+      summary: Get status of ETL associated with data_id
+      responses:
+        200:
+          description: OK
+          schema:
+            $ref: '#/definitions/ETLStatus'
+  /data/{data_id}/subsets:
+    parameters:
+      - name: data_id
+        in: path
+        description: ID given on launching an ETL
+        required: true
+        type: string
+    post:
+      summary: Create a new subset for the loaded data
+      parameters:
+        - name: subjects
+          in: query
+          description: List of subjects that define a new subset
+          required: true
+          type: string
+      responses:
+        201:
+          description: Created
+          schema:
+            type: object
+            properties:
+              subset_id:
+                type: integer
+  /data/{data_id}/subsets/{subset_id}:
+    parameters:
+      - name: data_id
+        in: path
+        description: ID given on launching an ETL
+        required: true
+        type: string
+      - name: subset_id
+        in: path
+        description: ID given when creating subset
+        required: true
+        type: string
+    delete:
+      summary: Delete subset definition
+      responses:
+        200:
+          description: OK
+  /analytics:
+    post:
+      summary: Launch analytical script
+      parameters:
+        - name: script
+          in: query
+          description: Analysis script to run
+          required: true
+          type: string
+        - name: arguments
+          in: query
+          description: String describing the arguments to the analysis script
+          required: true
+          type: string
+      responses:
+        201:
+          description: Created
+          schema:
+            type: object
+            properties:
+              job_id:
+                type: string
+  /analytics/{job_id}:
+    parameters:
+      - name: job_id
+        in: path
+        description: ID given on job creation
+        required: true
+        type: string
+    get:
+      summary: Get job status information for given job id
+      responses:
+        200:
+          description: OK
+          schema:
+            $ref: '#/definitions/JobStatus'
+    delete:
+      summary: Attempts to kill all processes associated with the job
+      responses:
+        200:
+          description: OK
+    
+definitions:
+  ETLStatus:
+    type: object
+    properties:
+      data_id:
+        type: string
+      status:
+        type: string
+  JobStatus:
+    type: object
+    properties:
+      status: 
+        type: string
+      warnings:
+        type: string
+      result:
+        type: string
+  Error:
+    type: object
+    properties:
+      error_msg:
+        type: string
\ No newline at end of file

fractalis/analytics/job.py

 from fractalis import celery_app
 
 
-def start_job():
+def create_job():
     pass
 
 

fractalis/analytics/scripts/test/sample.py

+from fractalis import celery_app
+
+
+@celery_app.task
+def add(a, b):
+    return a + b

fractalis/config.py

@@ -5,7 +5,7 @@ Exports:
 """
 import os
 
-from redislite import Redis
+from redislite import StrictRedis
 
 
 class BaseConfig(object):
@@ -13,9 +13,9 @@ class BaseConfig(object):
     DEBUG = False
     TESTING = False
     REDIS_DB_PATH = os.path.join(os.sep, 'tmp', 'fractalis.db')
-    rdb = Redis(REDIS_DB_PATH)
-    CELERY_BROKER_URL = 'redis+socket://{}'.format(rdb.socket_file)
-    CELERY_RESULT_BACKEND = 'redis+socket://{}'.format(rdb.socket_file)
+    redis = StrictRedis(REDIS_DB_PATH)
+    CELERY_BROKER_URL = 'redis+socket://{}'.format(redis.socket_file)
+    CELERY_RESULT_BACKEND = 'redis+socket://{}'.format(redis.socket_file)
 
 
 class DevelopmentConfig(BaseConfig):

fractalis/session.py

+from uuid import uuid4
+
+from redislite import StrictRedis
+
+from fractalis import flask_app
+from flask.sessions import SecureCookieSessionInterface, SecureCookieSession
+
+
+class RedisSession(SecureCookieSession):
+
+    def __init__(self, sid, initial=None):
+        super().__init__(initial=initial)
+        self.sid = sid
+
+
+class RedisSessionInterface(SecureCookieSessionInterface):
+
+    def __init__(self):
+        self.redis = StrictRedis(flask_app.config['REDIS_DB_PATH'])
+
+    def open_session(self, app, request):
+        sid = request.cookies.get(app.session_cookie_name)
+        if not sid:
+            sid = str(uuid4())
+            return RedisSession(sid=sid)
+        session_data = self.redis.get('session:{}'.format(sid))
+        if session_data is not None:
+            session_data = self.serializer.loads(session_data)
+            return RedisSession(initial=session_data, sid=sid)
+        return RedisSession(sid=sid)
+
+    def save_session(self, app, session, response):
+        domain = self.get_cookie_domain(app)
+        if not session:
+            self.redis.delete('session:{}'.format(session.sid))
+            if session.modified:
+                response.delete_cookie(app.session_cookie_name, domain=domain)
+            return None
+        session_expiration_time = self.get_expiration_time(app, session)
+        serialzed_session_data = self.serializer.dumps(dict(session))
+        self.redis.setex('session:{}'.format(session.sid),
+                         session_expiration_time, serialzed_session_data)
+        response.set_cookie(app.session_cookie_name, session.sid,
+                            expires=session_expiration_time, httponly=True,
+                            domain=domain)

tests/test_analytics.py

@@ -7,7 +7,7 @@ import pytest
 class TestAnalytics(object):
 
     @pytest.fixture
-    def test_client(self):
+    def flask_app(self):
         from flask import Flask
         from fractalis.config import configure_app
         app = Flask('test_app')
@@ -15,44 +15,64 @@ class TestAnalytics(object):
         test_client = app.test_client()
         return test_client
 
-    def test_create_new_resource(self, test_client):
-        response = test_client.post('/analytics')
+    def test_new_resource_created(self, flask_app):
+        response = flask_app.post('/analytics', data=dict(
+            script='test/sample.py',
+            arguments={'a': 1, 'b': 1}
+        ))
         response_body = json.loads(response.get_data().decode('utf-8'))
         new_resource_url = '/analytics/{}'.format(response_body['job_id'])
         assert response.status_code == 201
         assert uuid.UUID(response_body['job_id'])
-        assert test_client.head(new_resource_url).status_code == 200
+        assert flask_app.head(new_resource_url).status_code == 200
 
-    def test_create_new_resource_with_not_existing_script(self, test_client):
-        assert False
+    def test_400_if_creating_but_script_does_not_exist(self, flask_app):
+        response = flask_app.post('/analytics', data=dict(
+            script='test/sapmle.py',
+            arguments={'a': 1, 'b': 1}
+        ))
+        response_body = json.loads(response.get_data().decode('utf-8'))
+        assert response.status_code == 400
+        assert response_body['error_msg']
 
-    def test_create_new_resource_not_authenticated(self, test_client):
-        assert False
+    def test_400_if_creating_but_arguments_are_invalid(self, flask_app):
+        response = flask_app.post('/analytics', data=dict(
+            script='test/sample.py',
+            arguments={'a': 1, 'c': 1}
+        ))
+        response_body = json.loads(response.get_data().decode('utf-8'))
+        assert response.status_code == 400
+        assert response_body['error_msg']
 
-    def test_delete_resource(self, test_client):
-        response = test_client.post('/analytics')
+    def test_403_if_creating_but_not_authenticated(self, flask_app):
+        response = flask_app.post('/analytics', data=dict(
+            script='test/sample.py',
+            arguments={'a': 1, 'b': 1}
+        ))
         response_body = json.loads(response.get_data().decode('utf-8'))
-        new_resource_url = '/analytics/{}'.format(response_body['job_id'])
-        assert test_client.delete(new_resource_url).status_code == 200
-        assert test_client.head(new_resource_url).status_code == 404
+        assert response.status_code == 403
 
-    def test_delete_resource_not_authenticated(self, test_client):
-        assert False
+    def test_resource_deleted(self, flask_app):
+        response = flask_app.post('/analytics')
+        response_body = json.loads(response.get_data().decode('utf-8'))
+        new_resource_url = '/analytics/{}'.format(response_body['job_id'])
+        assert flask_app.delete(new_resource_url).status_code == 200
+        assert flask_app.head(new_resource_url).status_code == 404
 
-    def test_delete_not_existing_resource(self, test_client):
+    def test_403_if_deleting_but_not_authenticated(self, flask_app):
         assert False
 
-    def test_get_status_running(self, test_client):
+    def test_404_if_deleting_non_existing_resource(self, flask_app):
         assert False
 
-    def test_get_status_not_authenticated(self, test_client):
+    def test_403_when_getting_status_but_not_authenticated(self, flask_app):
         assert False
 
-    def test_get_status_if_script_failure(self, test_client):
+    def test_status_result_non_empty_if_finished(self, flask_app):
         assert False
 
-    def test_get_status_if_script_success(self, test_client):
+    def test_status_result_empty_if_not_finished(self, flask_app):
         assert False
 
-    def test_get_status_for_non_existing_resource(self, test_client):
+    def test_404_if_status_non_existing_resource(self, flask_app):
         assert False

tests/test_job.py

@@ -3,3 +3,26 @@ import pytest
 
 class TestJob(object):
 
+    def test_exception_when_starting_non_existing_script(self):
+        assert False
+
+    def test_start_job_returns_uuid(self):
+        assert False
+
+    def test_delayed_job_has_pending_status(self):
+        assert False
+
+    def test_job_in_progress_has_running_status(self):
+        assert False
+
+    def test_finished_job_returns_results(self):
+        assert False
+
+    def test_exception_when_checking_non_existing_job(self):
+        assert False
+
+    def test_job_is_gone_after_canceling(self):
+        assert False
+
+    def test_exception_when_canceling_non_existing_job(self):
+        assert False