From 90b612aa1267be134e79de9e19b2d65ed95be213 Mon Sep 17 00:00:00 2001
From: "sascha.herzinger" <sascha.herzinger@uni.lu>
Date: Fri, 2 Mar 2018 12:50:54 -0500
Subject: [PATCH] Fixing import manage issue in sdist
---
MANIFEST.in | 1 -
fractalis/cleanup.py | 17 +++++++++++++++++
fractalis/data/etlhandler.py | 4 ++--
manage.py | 17 +++--------------
.../etls/{transmart => }/test_etlhandler.py | 0
tests/unit/{test_manage.py => test_cleanup.py} | 6 +++---
6 files changed, 25 insertions(+), 20 deletions(-)
create mode 100644 fractalis/cleanup.py
rename tests/unit/etls/{transmart => }/test_etlhandler.py (100%)
rename tests/unit/{test_manage.py => test_cleanup.py} (91%)
diff --git a/MANIFEST.in b/MANIFEST.in
index 98b1c01..8da6dae 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1,2 +1 @@
include fractalis/logging.yaml
-include manage.py
diff --git a/fractalis/cleanup.py b/fractalis/cleanup.py
new file mode 100644
index 0000000..86c082b
--- /dev/null
+++ b/fractalis/cleanup.py
@@ -0,0 +1,17 @@
+import os
+
+from fractalis import app, redis, sync, celery
+
+
+@celery.task
+def janitor():
+ """Ideally this is maintained by a systemd service to cleanup redis and the
+ file system while Fractalis is running.
+ """
+ tmp_dir = app.config['FRACTALIS_TMP_DIR']
+ tracked_files = [key.split(':')[1] for key in redis.scan_iter('data:*')]
+ cached_files = [f for f in os.listdir(tmp_dir)
+ if os.path.isfile(os.path.join(tmp_dir, f))]
+ for cached_file in cached_files:
+ if cached_file not in tracked_files:
+ sync.remove_file(os.path.join(tmp_dir, cached_file))
diff --git a/fractalis/data/etlhandler.py b/fractalis/data/etlhandler.py
index 2006733..c1defa2 100644
--- a/fractalis/data/etlhandler.py
+++ b/fractalis/data/etlhandler.py
@@ -7,7 +7,7 @@ import logging
from uuid import uuid4
from typing import List, Union
-import manage
+from fractalis.cleanup import janitor
from fractalis import app, redis, celery
from fractalis.data.etl import ETL
@@ -136,7 +136,7 @@ class ETLHandler(metaclass=abc.ABCMeta):
task_ids = self.find_duplicates(data_tasks, descriptor)
for task_id in task_ids:
redis.delete('data:{}'.format(task_id))
- manage.janitor.delay()
+ janitor.delay()
def find_duplicate_task_id(self, data_tasks: List[str],
descriptor: dict) -> Union[str, None]:
diff --git a/manage.py b/manage.py
index 673fdf5..f5630c6 100644
--- a/manage.py
+++ b/manage.py
@@ -1,26 +1,15 @@
-import os
-
from flask_script import Manager
-from fractalis import app, redis, sync, celery
+import fractalis.cleanup
+from fractalis import app
manager = Manager(app)
-@celery.task
@manager.command
def janitor():
- """Ideally this is maintained by a systemd service to cleanup redis and the
- file system while Fractalis is running.
- """
- tmp_dir = app.config['FRACTALIS_TMP_DIR']
- tracked_files = [key.split(':')[1] for key in redis.scan_iter('data:*')]
- cached_files = [f for f in os.listdir(tmp_dir)
- if os.path.isfile(os.path.join(tmp_dir, f))]
- for cached_file in cached_files:
- if cached_file not in tracked_files:
- sync.remove_file(os.path.join(tmp_dir, cached_file))
+ fractalis.cleanup.janitor.delay()
if __name__ == "__main__":
diff --git a/tests/unit/etls/transmart/test_etlhandler.py b/tests/unit/etls/test_etlhandler.py
similarity index 100%
rename from tests/unit/etls/transmart/test_etlhandler.py
rename to tests/unit/etls/test_etlhandler.py
diff --git a/tests/unit/test_manage.py b/tests/unit/test_cleanup.py
similarity index 91%
rename from tests/unit/test_manage.py
rename to tests/unit/test_cleanup.py
index 52b9c35..05c9559 100644
--- a/tests/unit/test_manage.py
+++ b/tests/unit/test_cleanup.py
@@ -3,7 +3,7 @@
import os
from pathlib import Path
-import manage
+from fractalis.cleanup import janitor
from fractalis import app, redis
@@ -14,7 +14,7 @@ class TestManage:
tmp_dir = app.config['FRACTALIS_TMP_DIR']
os.makedirs(tmp_dir, exist_ok=True)
Path(os.path.join(tmp_dir, 'abc')).touch()
- manage.janitor()
+ janitor()
assert not os.path.exists(os.path.join(tmp_dir, 'abc'))
def test_janitor_does_not_remove_tracked_files(self):
@@ -22,5 +22,5 @@ class TestManage:
os.makedirs(tmp_dir, exist_ok=True)
Path(os.path.join(tmp_dir, 'abc')).touch()
redis.set('data:abc', '')
- manage.janitor()
+ janitor()
assert os.path.exists(os.path.join(tmp_dir, 'abc'))
--
GitLab