From 717010f2bd493495c3abc16ea9dc2b6f65b4cf20 Mon Sep 17 00:00:00 2001
From: Sascha Herzinger <sascha.herzinger@uni.lu>
Date: Thu, 9 Mar 2017 08:54:06 +0100
Subject: [PATCH] It is now possible to auth with token or credentials.
---
fractalis/data/controller.py | 10 ++---
fractalis/data/etlhandler.py | 32 ++++++++++++----
fractalis/data/etls/ada/etl_default.py | 18 +++++++++
fractalis/data/etls/ada/handler_ada.py | 4 ++
fractalis/data/etls/test/handler_test.py | 6 ++-
fractalis/data/etls/transmart/etl_hdd.py | 0
fractalis/data/etls/transmart/etl_ldd.py | 0
.../data/etls/transmart/handler_transmart.py | 9 -----
fractalis/data/schema.py | 12 +++++-
setup.py | 3 +-
tests/test_analytics.py | 2 +-
tests/test_data.py | 38 +++++++++++--------
12 files changed, 91 insertions(+), 43 deletions(-)
create mode 100644 fractalis/data/etls/ada/etl_default.py
delete mode 100644 fractalis/data/etls/transmart/etl_hdd.py
delete mode 100644 fractalis/data/etls/transmart/etl_ldd.py
delete mode 100644 fractalis/data/etls/transmart/handler_transmart.py
diff --git a/fractalis/data/controller.py b/fractalis/data/controller.py
index 7fab756..b7028b4 100644
--- a/fractalis/data/controller.py
+++ b/fractalis/data/controller.py
@@ -25,11 +25,11 @@ def prepare_session():
@validate_schema(create_data_schema)
def create_data():
wait = request.args.get('wait') == '1'
- json = request.get_json(force=True) # pattern enforced by decorators
- etlhandler = ETLHandler.factory(handler=json['handler'],
- server=json['server'],
- token=json['token'])
- data_ids = etlhandler.handle(descriptors=json['descriptors'], wait=wait)
+ payload = request.get_json(force=True) # pattern enforced by decorators
+ etl_handler = ETLHandler.factory(handler=payload['handler'],
+ server=payload['server'],
+ auth=payload['auth'])
+ data_ids = etl_handler.handle(descriptors=payload['descriptors'], wait=wait)
session['data_ids'] += data_ids
return jsonify({'data_ids': data_ids}), 201
diff --git a/fractalis/data/etlhandler.py b/fractalis/data/etlhandler.py
index 4856d63..ca34714 100644
--- a/fractalis/data/etlhandler.py
+++ b/fractalis/data/etlhandler.py
@@ -28,9 +28,25 @@ class ETLHandler(metaclass=abc.ABCMeta):
"""
pass
- def __init__(self, server, token):
+ @abc.abstractmethod
+ def _get_token_for_credentials(self, server: str,
+ user: str, passwd: str) -> str:
+ """ Authenticate with the server and return a token.
+
+ :param server: The server to authenticate with.
+ :param user: The user id.
+ :param passwd: The password.
+ """
+ pass
+
+ def __init__(self, server, auth):
self._server = server
- self._token = token
+ # if no token is given we have to get one
+ try:
+ self._token = auth['token']
+ except KeyError:
+ self._token = self._get_token_for_credentials(
+ server, auth['user'], auth['passwd'])
@staticmethod
def compute_data_id(server: str, descriptor: dict) -> str:
@@ -47,9 +63,9 @@ class ETLHandler(metaclass=abc.ABCMeta):
return hash_key
def handle(self, descriptors: List[dict], wait: bool = False) -> List[str]:
- """Create instances of ETL for the given descriptors and submit them (ETL
- implements celery.Task) to the broker. The task ids are returned to keep
- track of them.
+ """Create instances of ETL for the given descriptors and submit them
+ (ETL implements celery.Task) to the broker. The task ids are returned to
+ keep track of them.
:param descriptors: A list of items describing the data to download.
:param wait: Makes this method synchronous by waiting for the tasks to
@@ -86,20 +102,20 @@ class ETLHandler(metaclass=abc.ABCMeta):
return data_ids
@classmethod
- def factory(cls, handler: str, server: str, token: str) -> 'ETLHandler':
+ def factory(cls, handler: str, server: str, auth: dict) -> 'ETLHandler':
"""Return an instance of the implementation of ETLHandler that can
handle the given parameters.
:param handler: Describes the handler. E.g.: transmart, ada
:param server: The server to download data from.
- :param token: The token used for authentication.
+ :param auth: Contains credentials to authenticate with the API.
:return: An instance of an implementation of ETLHandler that returns
True for self.can_handle
"""
from . import HANDLER_REGISTRY
for Handler in HANDLER_REGISTRY:
if Handler.can_handle(handler):
- return Handler(server, token)
+ return Handler(server, auth)
raise NotImplementedError(
"No ETLHandler implementation found for: '{}'".format(handler))
diff --git a/fractalis/data/etls/ada/etl_default.py b/fractalis/data/etls/ada/etl_default.py
new file mode 100644
index 0000000..c24de74
--- /dev/null
+++ b/fractalis/data/etls/ada/etl_default.py
@@ -0,0 +1,18 @@
+"""Default ETL to get data from ADA"""
+
+from pandas import DataFrame
+
+from fractalis.data.etl import ETL
+
+
+class DefaultETL(ETL):
+
+ name = 'ada_default_etl'
+ _handler = 'ada'
+ _data_type = 'default'
+
+ def extract(self, server: str, token: str, descriptor: dict) -> object:
+ pass
+
+ def transform(self, raw_data: object) -> DataFrame:
+ pass
\ No newline at end of file
diff --git a/fractalis/data/etls/ada/handler_ada.py b/fractalis/data/etls/ada/handler_ada.py
index 3040ae4..9fef1cd 100644
--- a/fractalis/data/etls/ada/handler_ada.py
+++ b/fractalis/data/etls/ada/handler_ada.py
@@ -6,4 +6,8 @@ class AdaHandler(ETLHandler):
_handler = 'ada'
def _heartbeat(self):
+ pass
+
+ def _get_token_for_credentials(self, server: str,
+ user: str, passwd: str) -> str:
pass
\ No newline at end of file
diff --git a/fractalis/data/etls/test/handler_test.py b/fractalis/data/etls/test/handler_test.py
index 39ace18..b54d815 100644
--- a/fractalis/data/etls/test/handler_test.py
+++ b/fractalis/data/etls/test/handler_test.py
@@ -5,5 +5,9 @@ class TestHandler(ETLHandler):
_handler = 'test'
- def _heartbeat():
+ def _heartbeat(self):
+ pass
+
+ def _get_token_for_credentials(self, server: str,
+ user: str, passwd: str) -> str:
pass
diff --git a/fractalis/data/etls/transmart/etl_hdd.py b/fractalis/data/etls/transmart/etl_hdd.py
deleted file mode 100644
index e69de29..0000000
diff --git a/fractalis/data/etls/transmart/etl_ldd.py b/fractalis/data/etls/transmart/etl_ldd.py
deleted file mode 100644
index e69de29..0000000
diff --git a/fractalis/data/etls/transmart/handler_transmart.py b/fractalis/data/etls/transmart/handler_transmart.py
deleted file mode 100644
index 88a9749..0000000
--- a/fractalis/data/etls/transmart/handler_transmart.py
+++ /dev/null
@@ -1,9 +0,0 @@
-from fractalis.data.etlhandler import ETLHandler
-
-
-class TransmartHandler(ETLHandler):
-
- _handler = 'transmart'
-
- def heartbeat():
- pass
diff --git a/fractalis/data/schema.py b/fractalis/data/schema.py
index 557f129..cfc3906 100644
--- a/fractalis/data/schema.py
+++ b/fractalis/data/schema.py
@@ -3,7 +3,15 @@ create_data_schema = {
"properties": {
"handler": {"type": "string"},
"server": {"type": "string"},
- "token": {"type": "string"},
+ "auth": {
+ "type": "object",
+ "properties": {
+ "token": {"type": "string"},
+ "user": {"type": "string"},
+ "passwd": {"type": "string"}
+ },
+ "minProperties": 1
+ },
"descriptors": {
"type": "array",
"items": {
@@ -16,5 +24,5 @@ create_data_schema = {
}
}
},
- "required": ["handler", "server", "token", "descriptors"]
+ "required": ["handler", "server", "auth", "descriptors"]
}
diff --git a/setup.py b/setup.py
index cc5a0d6..b06c8e8 100644
--- a/setup.py
+++ b/setup.py
@@ -12,7 +12,8 @@ setup(
'celery[redis]',
'redis',
'pandas',
- 'numpy'
+ 'numpy',
+ 'requests'
],
setup_requires=[
'pytest-runner',
diff --git a/tests/test_analytics.py b/tests/test_analytics.py
index c3ad178..30b1780 100644
--- a/tests/test_analytics.py
+++ b/tests/test_analytics.py
@@ -31,7 +31,7 @@ class TestAnalytics:
'/data', data=flask.json.dumps(dict(
handler='test',
server='localhost:1234',
- token='7746391376142672192764',
+ auth={'token': '7746391376142672192764'},
descriptors=[
{
'data_type': 'randomdf',
diff --git a/tests/test_data.py b/tests/test_data.py
index b3565fd..61d6dd2 100644
--- a/tests/test_data.py
+++ b/tests/test_data.py
@@ -29,7 +29,7 @@ class TestData:
'/data', data=flask.json.dumps(dict(
handler='test',
server='localhost:1234',
- token='7746391376142672192764',
+ auth={'token': '7746391376142672192764'},
descriptors=[
{
'data_type': 'foo',
@@ -44,7 +44,7 @@ class TestData:
'/data', data=flask.json.dumps(dict(
handler='test',
server='localhost:1234',
- token='7746391376142672192764',
+ auth={'token': '7746391376142672192764'},
descriptors=[
{
'data_type': 'foo',
@@ -65,63 +65,69 @@ class TestData:
{
'handler': '',
'server': 'localhost',
- 'token': '1234567890',
+ 'auth': '{"''tok"n'"" '12345678"90',
'descriptors': '[{"data_type": "foo", "concept": "GSE1234"}]'
},
{
'handler': 'test',
'server': '',
- 'token': '1234567890',
+ 'auth': '{"token": "1234567890"}',
'descriptors': '[{"data_type": "foo", "concept": "GSE1234"}]'
},
{
'handler': 'test',
'server': 'localhost',
- 'token': '',
+ 'auth': '',
'descriptors': '[{"data_type": "foo", "concept": "GSE1234"}]'
},
{
'handler': 'test',
'server': 'localhost',
- 'token': '1234567890',
+ 'auth': '{"token": "1234567890"}',
'descriptors': ''
},
{
'handler': 'test',
'server': 'localhost',
- 'token': '1234567890',
+ 'auth': '{"token": "1234567890"}',
'descriptors': '[{"data_type": "foo", "concept": "GSE1234"}]'
},
{
'handler': 'test',
'server': 'localhost',
- 'token': '1234567890',
+ 'auth': '{"token": "1234567890"}',
'descriptors': '[{"concept": "GSE1234"}]'
},
{
'handler': 'test',
'server': 'localhost',
- 'token': '1234567890',
+ 'auth': '{"token": "1234567890"}',
'descriptors': '[{"data_type": "foo"}]'
},
{
'handler': 'test',
'server': 'localhost',
- 'token': '1234567890',
+ 'auth': '{"token": "1234567890"}',
'descriptors': '[{"data_type": "", "concept": "GSE1234"}]'
},
{
'handler': 'test',
'server': 'localhost',
- 'token': '1234567890',
+ 'auth': '{"token": "234567890"}',
'descriptors': '[]'
+ },
+ {
+ 'handler': 'test',
+ 'server': 'localhost',
+ 'auth': '{}',
+ 'descriptors': '[{"data_type": "foo", "concept": "GSE1234"}]'
}
])
def bad_post(self, test_client, request):
return lambda: test_client.post('/data', data=flask.json.dumps(dict(
handler=request.param['handler'],
server=request.param['server'],
- token=request.param['token'],
+ auth=request.param['auth'],
descriptors=request.param['descriptors']
)))
@@ -273,7 +279,7 @@ class TestData:
data = dict(
handler='test',
server='localhost:1234',
- token='7746391376142672192764',
+ auth={'token': '7746391376142672192764'},
descriptors=[
{
'data_type': 'foo',
@@ -320,7 +326,7 @@ class TestData:
data = dict(
handler='test',
server='localhost:1234',
- token='7746391376142672192764',
+ auth={'token': '7746391376142672192764'},
descriptors=[
{
'data_type': 'foo',
@@ -361,7 +367,7 @@ class TestData:
data = dict(
handler='abc',
server='localhost:1234',
- token='7746391376142672192764',
+ auth={'token': '7746391376142672192764'},
descriptors=[
{
'data_type': 'foo',
@@ -376,7 +382,7 @@ class TestData:
data = dict(
handler='test',
server='localhost:1234',
- token='7746391376142672192764',
+ auth={'token': '7746391376142672192764'},
descriptors=[
{
'data_type': 'abc',
--
GitLab