diff --git a/fractalis/data/controller.py b/fractalis/data/controller.py
index 7fab756cfd388099f2001bca82d26855788c3fac..b7028b4a4bc850e85db2431e1f4c1024af9bed05 100644
--- a/fractalis/data/controller.py
+++ b/fractalis/data/controller.py
@@ -25,11 +25,11 @@ def prepare_session():
@validate_schema(create_data_schema)
def create_data():
wait = request.args.get('wait') == '1'
- json = request.get_json(force=True) # pattern enforced by decorators
- etlhandler = ETLHandler.factory(handler=json['handler'],
- server=json['server'],
- token=json['token'])
- data_ids = etlhandler.handle(descriptors=json['descriptors'], wait=wait)
+ payload = request.get_json(force=True) # pattern enforced by decorators
+ etl_handler = ETLHandler.factory(handler=payload['handler'],
+ server=payload['server'],
+ auth=payload['auth'])
+ data_ids = etl_handler.handle(descriptors=payload['descriptors'], wait=wait)
session['data_ids'] += data_ids
return jsonify({'data_ids': data_ids}), 201
diff --git a/fractalis/data/etlhandler.py b/fractalis/data/etlhandler.py
index 4856d6372035c5a040f2be5c43012493e41bf45d..ca347144be10eb21304f78614e196172399f1cc1 100644
--- a/fractalis/data/etlhandler.py
+++ b/fractalis/data/etlhandler.py
@@ -28,9 +28,25 @@ class ETLHandler(metaclass=abc.ABCMeta):
"""
pass
- def __init__(self, server, token):
+ @abc.abstractmethod
+ def _get_token_for_credentials(self, server: str,
+ user: str, passwd: str) -> str:
+ """ Authenticate with the server and return a token.
+
+ :param server: The server to authenticate with.
+ :param user: The user id.
+ :param passwd: The password.
+ """
+ pass
+
+ def __init__(self, server, auth):
self._server = server
- self._token = token
+ # if no token is given we have to get one
+ try:
+ self._token = auth['token']
+ except KeyError:
+ self._token = self._get_token_for_credentials(
+ server, auth['user'], auth['passwd'])
@staticmethod
def compute_data_id(server: str, descriptor: dict) -> str:
@@ -47,9 +63,9 @@ class ETLHandler(metaclass=abc.ABCMeta):
return hash_key
def handle(self, descriptors: List[dict], wait: bool = False) -> List[str]:
- """Create instances of ETL for the given descriptors and submit them (ETL
- implements celery.Task) to the broker. The task ids are returned to keep
- track of them.
+ """Create instances of ETL for the given descriptors and submit them
+ (ETL implements celery.Task) to the broker. The task ids are returned to
+ keep track of them.
:param descriptors: A list of items describing the data to download.
:param wait: Makes this method synchronous by waiting for the tasks to
@@ -86,20 +102,20 @@ class ETLHandler(metaclass=abc.ABCMeta):
return data_ids
@classmethod
- def factory(cls, handler: str, server: str, token: str) -> 'ETLHandler':
+ def factory(cls, handler: str, server: str, auth: dict) -> 'ETLHandler':
"""Return an instance of the implementation of ETLHandler that can
handle the given parameters.
:param handler: Describes the handler. E.g.: transmart, ada
:param server: The server to download data from.
- :param token: The token used for authentication.
+ :param auth: Contains credentials to authenticate with the API.
:return: An instance of an implementation of ETLHandler that returns
True for self.can_handle
"""
from . import HANDLER_REGISTRY
for Handler in HANDLER_REGISTRY:
if Handler.can_handle(handler):
- return Handler(server, token)
+ return Handler(server, auth)
raise NotImplementedError(
"No ETLHandler implementation found for: '{}'".format(handler))
diff --git a/fractalis/data/etls/ada/etl_default.py b/fractalis/data/etls/ada/etl_default.py
new file mode 100644
index 0000000000000000000000000000000000000000..c24de748e498438072b9651907558a57a7388e76
--- /dev/null
+++ b/fractalis/data/etls/ada/etl_default.py
@@ -0,0 +1,18 @@
+"""Default ETL to get data from ADA"""
+
+from pandas import DataFrame
+
+from fractalis.data.etl import ETL
+
+
+class DefaultETL(ETL):
+
+ name = 'ada_default_etl'
+ _handler = 'ada'
+ _data_type = 'default'
+
+ def extract(self, server: str, token: str, descriptor: dict) -> object:
+ pass
+
+ def transform(self, raw_data: object) -> DataFrame:
+ pass
\ No newline at end of file
diff --git a/fractalis/data/etls/ada/handler_ada.py b/fractalis/data/etls/ada/handler_ada.py
index 3040ae4dcfa1b9b0f887a7572e1790eb33f98f0d..9fef1cdb01b2618ec562aaea8265cddce37d248b 100644
--- a/fractalis/data/etls/ada/handler_ada.py
+++ b/fractalis/data/etls/ada/handler_ada.py
@@ -6,4 +6,8 @@ class AdaHandler(ETLHandler):
_handler = 'ada'
def _heartbeat(self):
+ pass
+
+ def _get_token_for_credentials(self, server: str,
+ user: str, passwd: str) -> str:
pass
\ No newline at end of file
diff --git a/fractalis/data/etls/test/handler_test.py b/fractalis/data/etls/test/handler_test.py
index 39ace185fddab97347c42b409fe272e9dec05af6..b54d815051fccec76e6f9fe697d35e0f0b92560f 100644
--- a/fractalis/data/etls/test/handler_test.py
+++ b/fractalis/data/etls/test/handler_test.py
@@ -5,5 +5,9 @@ class TestHandler(ETLHandler):
_handler = 'test'
- def _heartbeat():
+ def _heartbeat(self):
+ pass
+
+ def _get_token_for_credentials(self, server: str,
+ user: str, passwd: str) -> str:
pass
diff --git a/fractalis/data/etls/transmart/etl_hdd.py b/fractalis/data/etls/transmart/etl_hdd.py
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/fractalis/data/etls/transmart/etl_ldd.py b/fractalis/data/etls/transmart/etl_ldd.py
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/fractalis/data/etls/transmart/handler_transmart.py b/fractalis/data/etls/transmart/handler_transmart.py
deleted file mode 100644
index 88a9749ce166b6dff4a399f254fe6e601addd7cc..0000000000000000000000000000000000000000
--- a/fractalis/data/etls/transmart/handler_transmart.py
+++ /dev/null
@@ -1,9 +0,0 @@
-from fractalis.data.etlhandler import ETLHandler
-
-
-class TransmartHandler(ETLHandler):
-
- _handler = 'transmart'
-
- def heartbeat():
- pass
diff --git a/fractalis/data/schema.py b/fractalis/data/schema.py
index 557f1291a06d8fe840914a8adec18e001c3e30b0..cfc390667307c862e905764114628d5605b5bf1e 100644
--- a/fractalis/data/schema.py
+++ b/fractalis/data/schema.py
@@ -3,7 +3,15 @@ create_data_schema = {
"properties": {
"handler": {"type": "string"},
"server": {"type": "string"},
- "token": {"type": "string"},
+ "auth": {
+ "type": "object",
+ "properties": {
+ "token": {"type": "string"},
+ "user": {"type": "string"},
+ "passwd": {"type": "string"}
+ },
+ "minProperties": 1
+ },
"descriptors": {
"type": "array",
"items": {
@@ -16,5 +24,5 @@ create_data_schema = {
}
}
},
- "required": ["handler", "server", "token", "descriptors"]
+ "required": ["handler", "server", "auth", "descriptors"]
}
diff --git a/setup.py b/setup.py
index cc5a0d6b7e2385d1835991a4370e4827d8a1c2d7..b06c8e8722a3e3898030cc970323f964cd4276ae 100644
--- a/setup.py
+++ b/setup.py
@@ -12,7 +12,8 @@ setup(
'celery[redis]',
'redis',
'pandas',
- 'numpy'
+ 'numpy',
+ 'requests'
],
setup_requires=[
'pytest-runner',
diff --git a/tests/test_analytics.py b/tests/test_analytics.py
index c3ad178db1585400247fb040d3fbca9b47cfd86d..30b178043b656ae0ea37cceb3016cdca07801d9b 100644
--- a/tests/test_analytics.py
+++ b/tests/test_analytics.py
@@ -31,7 +31,7 @@ class TestAnalytics:
'/data', data=flask.json.dumps(dict(
handler='test',
server='localhost:1234',
- token='7746391376142672192764',
+ auth={'token': '7746391376142672192764'},
descriptors=[
{
'data_type': 'randomdf',
diff --git a/tests/test_data.py b/tests/test_data.py
index b3565fdef388a5b7f3ba3c024274127d890b347f..61d6dd2a9904ee54e58cd21e76db334bbe4561dd 100644
--- a/tests/test_data.py
+++ b/tests/test_data.py
@@ -29,7 +29,7 @@ class TestData:
'/data', data=flask.json.dumps(dict(
handler='test',
server='localhost:1234',
- token='7746391376142672192764',
+ auth={'token': '7746391376142672192764'},
descriptors=[
{
'data_type': 'foo',
@@ -44,7 +44,7 @@ class TestData:
'/data', data=flask.json.dumps(dict(
handler='test',
server='localhost:1234',
- token='7746391376142672192764',
+ auth={'token': '7746391376142672192764'},
descriptors=[
{
'data_type': 'foo',
@@ -65,63 +65,69 @@ class TestData:
{
'handler': '',
'server': 'localhost',
- 'token': '1234567890',
+ 'auth': '{"''tok"n'"" '12345678"90',
'descriptors': '[{"data_type": "foo", "concept": "GSE1234"}]'
},
{
'handler': 'test',
'server': '',
- 'token': '1234567890',
+ 'auth': '{"token": "1234567890"}',
'descriptors': '[{"data_type": "foo", "concept": "GSE1234"}]'
},
{
'handler': 'test',
'server': 'localhost',
- 'token': '',
+ 'auth': '',
'descriptors': '[{"data_type": "foo", "concept": "GSE1234"}]'
},
{
'handler': 'test',
'server': 'localhost',
- 'token': '1234567890',
+ 'auth': '{"token": "1234567890"}',
'descriptors': ''
},
{
'handler': 'test',
'server': 'localhost',
- 'token': '1234567890',
+ 'auth': '{"token": "1234567890"}',
'descriptors': '[{"data_type": "foo", "concept": "GSE1234"}]'
},
{
'handler': 'test',
'server': 'localhost',
- 'token': '1234567890',
+ 'auth': '{"token": "1234567890"}',
'descriptors': '[{"concept": "GSE1234"}]'
},
{
'handler': 'test',
'server': 'localhost',
- 'token': '1234567890',
+ 'auth': '{"token": "1234567890"}',
'descriptors': '[{"data_type": "foo"}]'
},
{
'handler': 'test',
'server': 'localhost',
- 'token': '1234567890',
+ 'auth': '{"token": "1234567890"}',
'descriptors': '[{"data_type": "", "concept": "GSE1234"}]'
},
{
'handler': 'test',
'server': 'localhost',
- 'token': '1234567890',
+ 'auth': '{"token": "234567890"}',
'descriptors': '[]'
+ },
+ {
+ 'handler': 'test',
+ 'server': 'localhost',
+ 'auth': '{}',
+ 'descriptors': '[{"data_type": "foo", "concept": "GSE1234"}]'
}
])
def bad_post(self, test_client, request):
return lambda: test_client.post('/data', data=flask.json.dumps(dict(
handler=request.param['handler'],
server=request.param['server'],
- token=request.param['token'],
+ auth=request.param['auth'],
descriptors=request.param['descriptors']
)))
@@ -273,7 +279,7 @@ class TestData:
data = dict(
handler='test',
server='localhost:1234',
- token='7746391376142672192764',
+ auth={'token': '7746391376142672192764'},
descriptors=[
{
'data_type': 'foo',
@@ -320,7 +326,7 @@ class TestData:
data = dict(
handler='test',
server='localhost:1234',
- token='7746391376142672192764',
+ auth={'token': '7746391376142672192764'},
descriptors=[
{
'data_type': 'foo',
@@ -361,7 +367,7 @@ class TestData:
data = dict(
handler='abc',
server='localhost:1234',
- token='7746391376142672192764',
+ auth={'token': '7746391376142672192764'},
descriptors=[
{
'data_type': 'foo',
@@ -376,7 +382,7 @@ class TestData:
data = dict(
handler='test',
server='localhost:1234',
- token='7746391376142672192764',
+ auth={'token': '7746391376142672192764'},
descriptors=[
{
'data_type': 'abc',