From 94636db1c4e260263718cf054e405b37bfd03638 Mon Sep 17 00:00:00 2001
From: Sascha Herzinger <sascha.herzinger@uni.lu>
Date: Thu, 9 Mar 2017 07:38:47 +0100
Subject: [PATCH] Implemented sync tests and fixed 2 uncovered bugs. I love
tests.
---
fractalis/sync.py | 11 ++++--
tests/test_sync.py | 93 ++++++++++++++++++++++++++++++++++++++++++----
2 files changed, 93 insertions(+), 11 deletions(-)
diff --git a/fractalis/sync.py b/fractalis/sync.py
index 9539ca8..c45c419 100644
--- a/fractalis/sync.py
+++ b/fractalis/sync.py
@@ -16,7 +16,9 @@ from fractalis import app
@celery.task
def remove_expired_redis_entries() -> None:
- """Remove expired entries from the redis DB."""
+ """Remove 'data' entries from the redis DB for which 'last_access' lays back
+ longer than the timedelta defined in 'FRACTALIS_CACHE_EXP'.
+ """
cache_expr = app.config['FRACTALIS_CACHE_EXP']
redis_data = redis.hgetall(name='data')
for key in redis_data:
@@ -34,8 +36,8 @@ def remove_untracked_data_files() -> None:
"""Remove files that have no record in the redis DB"""
tmp_dir = app.config['FRACTALIS_TMP_DIR']
data_dir = os.path.join(tmp_dir, 'data')
- redis_data = redis.hdel('data')
- for file_path in iglob(os.path.join(data_dir, '*.py')):
+ redis_data = redis.hgetall('data')
+ for file_path in iglob(os.path.join(data_dir, '*')):
# check if file tracked by redis
is_tracked = False
for key in redis_data:
@@ -49,7 +51,8 @@ def remove_untracked_data_files() -> None:
def cleanup_all() -> None:
"""Reset redis and the filesystem. This is only useful for testing and
- should !!!NEVER!!! be used otherwise."""
+ should !!!NEVER!!! be used otherwise.
+ """
redis.flushall()
tmp_dir = app.config['FRACTALIS_TMP_DIR']
if os.path.exists(tmp_dir):
diff --git a/tests/test_sync.py b/tests/test_sync.py
index 57153e7..d9f373f 100644
--- a/tests/test_sync.py
+++ b/tests/test_sync.py
@@ -2,16 +2,95 @@
components synchronized.
"""
-from fractalis import app, redis, sync
+import os
+import json
+import time
+from pathlib import Path
+from uuid import uuid4
+
+import pytest
+
+from fractalis import app, sync
class TestSync:
- def test_expired_entries_removed(self):
- assert False
+ @pytest.fixture()
+ def redis(self):
+ from fractalis import redis
+ yield redis
+ sync.cleanup_all()
+
+ def test_expired_entries_removed(self, redis):
+ data_obj = {
+ 'file_path': 'foo',
+ 'last_access': 0 # last access = 1970
+ }
+ redis.hset(name='data', key=123, value=json.dumps(data_obj))
+ assert redis.hget(name='data', key=123)
+ sync.remove_expired_redis_entries()
+ assert not redis.hget(name='data', key=123)
+
+ def test_not_expired_entries_not_removed(self, redis):
+ data_obj = {
+ 'file_path': 'foo',
+ 'last_access': time.time() # now
+ }
+ redis.hset(name='data', key=123, value=json.dumps(data_obj))
+ assert redis.hget(name='data', key=123)
+ sync.remove_expired_redis_entries()
+ assert redis.hget(name='data', key=123)
+
+ def test_expired_tracked_files_removed(self, redis):
+ tmp_dir = app.config['FRACTALIS_TMP_DIR']
+ data_dir = os.path.join(tmp_dir, 'data')
+ file_path = os.path.join(data_dir, str(uuid4()))
+ data_obj = {
+ 'file_path': file_path,
+ 'last_access': 0 # last access = 1970
+ }
+ os.makedirs(data_dir, exist_ok=True)
+ Path(file_path).touch()
+ redis.hset(name='data', key=123, value=json.dumps(data_obj))
+ assert os.path.exists(file_path)
+ sync.remove_expired_redis_entries()
+ assert not os.path.exists(file_path)
+
+ def test_not_expired_tracked_files_not_removed(self, redis):
+ tmp_dir = app.config['FRACTALIS_TMP_DIR']
+ data_dir = os.path.join(tmp_dir, 'data')
+ file_path = os.path.join(data_dir, str(uuid4()))
+ data_obj = {
+ 'file_path': file_path,
+ 'last_access': time.time()
+ }
+ os.makedirs(data_dir, exist_ok=True)
+ Path(file_path).touch()
+ redis.hset(name='data', key=123, value=json.dumps(data_obj))
+ assert os.path.exists(file_path)
+ sync.remove_expired_redis_entries()
+ assert os.path.exists(file_path)
- def test_tracked_files_removed_after_redis_cleanup(self):
- assert False
+ def test_untracked_files_removed(self):
+ tmp_dir = app.config['FRACTALIS_TMP_DIR']
+ data_dir = os.path.join(tmp_dir, 'data')
+ file_path = os.path.join(data_dir, str(uuid4()))
+ os.makedirs(data_dir, exist_ok=True)
+ Path(file_path).touch()
+ assert os.path.exists(file_path)
+ sync.remove_untracked_data_files()
+ assert not os.path.exists(file_path)
- def test_untracked_removed(self):
- assert False
\ No newline at end of file
+ def test_tracked_files_not_removed(self, redis):
+ tmp_dir = app.config['FRACTALIS_TMP_DIR']
+ data_dir = os.path.join(tmp_dir, 'data')
+ file_path = os.path.join(data_dir, str(uuid4()))
+ data_obj = {
+ 'file_path': file_path
+ }
+ os.makedirs(data_dir, exist_ok=True)
+ Path(file_path).touch()
+ redis.hset(name='data', key=123, value=json.dumps(data_obj))
+ assert os.path.exists(file_path)
+ sync.remove_untracked_data_files()
+ assert os.path.exists(file_path)
--
GitLab