From 8e0f9b4e838cf177095f4cde96ead1668f7746e5 Mon Sep 17 00:00:00 2001
From: Piotr Gawron <piotr.gawron@uni.lu>
Date: Wed, 18 Nov 2020 08:59:34 +0100
Subject: [PATCH] form that allows editing visit import data
---
smash/web/forms/visit_import_data_form.py | 12 +++
.../importer/csv_tns_visit_import_reader.py | 5 ++
smash/web/importer/log_storage.py | 15 ++++
smash/web/importer/warning_counter.py | 11 +--
smash/web/templates/study/edit.html | 50 ++++++++++++
.../web/templates/visit_import_data/edit.html | 81 +++++++++++++++++++
smash/web/urls.py | 5 ++
smash/web/views/study.py | 61 +++++++++++++-
8 files changed, 233 insertions(+), 7 deletions(-)
create mode 100644 smash/web/forms/visit_import_data_form.py
create mode 100644 smash/web/importer/log_storage.py
create mode 100644 smash/web/templates/visit_import_data/edit.html
diff --git a/smash/web/forms/visit_import_data_form.py b/smash/web/forms/visit_import_data_form.py
new file mode 100644
index 00000000..8da98260
--- /dev/null
+++ b/smash/web/forms/visit_import_data_form.py
@@ -0,0 +1,12 @@
+from django.forms import ModelForm
+
+from web.models import VisitImportData
+
+
+class VisitImportDataEditForm(ModelForm):
+ class Meta:
+ model = VisitImportData
+ fields = '__all__'
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
diff --git a/smash/web/importer/csv_tns_visit_import_reader.py b/smash/web/importer/csv_tns_visit_import_reader.py
index 2c5ed625..ea2aa27a 100644
--- a/smash/web/importer/csv_tns_visit_import_reader.py
+++ b/smash/web/importer/csv_tns_visit_import_reader.py
@@ -194,6 +194,11 @@ class TnsCsvVisitImportReader:
try:
visit_number = data[self.visit_import_data.visit_number_column_name]
visit_number = int(visit_number) + (1 - self.visit_import_data.study.redcap_first_visit_number)
+ if visit_number < 1:
+ logger.warning(
+ "Visit number is invalid. Visit number should start from: " +
+ str(self.visit_import_data.study.redcap_first_visit_number) + ".")
+ visit_number = 1
return visit_number
except KeyError as e:
raise EtlException('Visit number is not defined') from e
diff --git a/smash/web/importer/log_storage.py b/smash/web/importer/log_storage.py
new file mode 100644
index 00000000..bdc78d22
--- /dev/null
+++ b/smash/web/importer/log_storage.py
@@ -0,0 +1,15 @@
+import logging
+
+
+class LogStorageHandler(logging.Handler):
+ level_messages = {}
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.level_messages = {}
+
+ def emit(self, record):
+ level = record.levelname
+ if level not in self.level_messages:
+ self.level_messages[level] = []
+ self.level_messages[level].append(self.format(record))
diff --git a/smash/web/importer/warning_counter.py b/smash/web/importer/warning_counter.py
index 0098a08f..d3755c45 100644
--- a/smash/web/importer/warning_counter.py
+++ b/smash/web/importer/warning_counter.py
@@ -1,14 +1,15 @@
import logging
+
class MsgCounterHandler(logging.Handler):
level2count = None
def __init__(self, *args, **kwargs):
- super(MsgCounterHandler, self).__init__(*args, **kwargs)
+ super().__init__(*args, **kwargs)
self.level2count = {}
def emit(self, record):
- l = record.levelname
- if l not in self.level2count:
- self.level2count[l] = 0
- self.level2count[l] += 1
+ level = record.levelname
+ if level not in self.level2count:
+ self.level2count[level] = 0
+ self.level2count[level] += 1
diff --git a/smash/web/templates/study/edit.html b/smash/web/templates/study/edit.html
index 5808f338..1512351c 100644
--- a/smash/web/templates/study/edit.html
+++ b/smash/web/templates/study/edit.html
@@ -94,6 +94,56 @@
</div>
</div><!-- /.box-body -->
+ <div class="box-header with-border">
+ <h3>ETL</h3>
+ </div>
+ <div class="box-body">
+ <table class="table table-bordered table-striped">
+ <thead>
+ <tr>
+ <th class="text-center">Action Type</th>
+ <th class="text-center">File</th>
+ <th class="text-center">File Type</th>
+ <th class="text-center">Run at</th>
+ <th class="text-center">Worker</th>
+ <th class="text-center">Edit</th>
+ <th class="text-center">Run manually</th>
+ </tr>
+ </thead>
+ <tbody>
+ {% for etl_entry in etl_entries %}
+ <tr>
+ <td class="text-center">{{ etl_entry.type }}</td>
+ <td class="text-center">{{ etl_entry.file }}</td>
+ <td class="text-center">{{ etl_entry.filetype }}</td>
+ <td class="text-center">{{ etl_entry.run_at }}</td>
+ <td class="text-center">{{ etl_entry.worker }}</td>
+ <td class="text-center"><a title="Edit ETL"
+ {% if etl_entry.type == 'Import visit' %}
+ href="{% url 'web.views.import_visit_edit' study_id etl_entry.id %}"
+ {% else %}
+ href="#"
+ {% endif %}
+ type="button"
+ class="btn btn-block btn-default">EDIT</a>
+ </td>
+ <td class="text-center">
+ {% if etl_entry.available %}
+ <a href="{% url 'web.views.import_visit_execute' study_id etl_entry.id %}"
+ type="button"
+ class="btn btn-block btn-default">Run</a>
+ {% else %}
+ <a type="button"
+ class="btn btn-block btn-warning" disabled>Unavailable</a>
+
+ {% endif %}
+ </td>
+ </tr>
+ {% endfor %}
+ </tbody>
+ </table>
+ </div>
+
{% include 'includes/custom_study_subject_field_box.html' with study=study fields=study.customstudysubjectfield_set.all %}
<div class="box-header with-border">
diff --git a/smash/web/templates/visit_import_data/edit.html b/smash/web/templates/visit_import_data/edit.html
new file mode 100644
index 00000000..ad50e6e8
--- /dev/null
+++ b/smash/web/templates/visit_import_data/edit.html
@@ -0,0 +1,81 @@
+{% extends "_base.html" %}
+{% load static %}
+{% load filters %}
+
+{% block styles %}
+ {{ block.super }}
+ <link rel="stylesheet" href="{% static 'AdminLTE/plugins/awesomplete/awesomplete.css' %}"/>
+
+ {% include "includes/datepicker.css.html" %}
+{% endblock styles %}
+
+{% block ui_active_tab %}'subjects'{% endblock ui_active_tab %}
+{% block page_header %}Edit visit import data{% endblock page_header %}
+{% block page_description %}{% endblock page_description %}
+
+{% block title %}{{ block.super }} - Edit visit import data{% endblock %}
+
+{% block breadcrumb %}
+ {% include "subjects/breadcrumb.html" %}
+{% endblock breadcrumb %}
+
+{% block maincontent %}
+
+ {% block content %}
+ <div class="row">
+ <div class="col-md-12">
+ <div class="box box-success">
+ <div class="box-header with-border">
+ <h3 class="box-title">Enter visit import details</h3>
+ </div>
+
+
+ <form method="post" action="" class="form-horizontal">
+ {% csrf_token %}
+
+ <div class="box-body">
+ {% for field in form %}
+ <div class="form-group {% if field.errors %}has-error{% endif %}">
+ <label class="col-sm-4 col-lg-offset-1 col-lg-2 control-label">
+ {{ field.label }}
+ </label>
+
+ <div class="col-sm-8 col-lg-4">
+ {{ field|add_class:'form-control' }}
+ </div>
+
+ {% if field.errors %}
+ <span class="help-block">
+ {{ field.errors }}
+ </span>
+ {% endif %}
+ </div>
+ {% endfor %}
+ </div><!-- /.box-body -->
+ <div class="box-footer">
+ <div class="col-sm-6">
+ <button type="submit" class="btn btn-block btn-success">Save</button>
+ </div>
+ <div class="col-sm-6">
+ <a href="{% url 'web.views.edit_study' study_id %}"
+ class="btn btn-block btn-default">Cancel</a>
+ </div>
+ </div><!-- /.box-footer -->
+ </form>
+ </div>
+
+ </div>
+ </div>
+
+ {% endblock %}
+
+
+{% endblock maincontent %}
+
+{% block scripts %}
+ {{ block.super }}
+
+ <script src="{% static 'AdminLTE/plugins/awesomplete/awesomplete.min.js' %}"></script>
+
+ {% include "includes/datetimepicker.js.html" %}
+{% endblock scripts %}
diff --git a/smash/web/urls.py b/smash/web/urls.py
index 7e5e1239..dcf6a803 100644
--- a/smash/web/urls.py
+++ b/smash/web/urls.py
@@ -253,6 +253,11 @@ urlpatterns = [
url(r'^study/(?P<study_id>\d+)/custom_study_subject_field/(?P<field_id>\d+)/delete',
views.study.custom_study_subject_field_delete, name='web.views.custom_study_subject_field_delete'),
+ url(r'^study/(?P<study_id>\d+)/import_visit_edit/(?P<import_id>\d+)/edit',
+ views.study.import_visit_edit, name='web.views.import_visit_edit'),
+ url(r'^study/(?P<study_id>\d+)/import_visit_edit/(?P<import_id>\d+)/execute',
+ views.study.import_visit_execute, name='web.views.import_visit_execute'),
+
####################
# EXPORT #
####################
diff --git a/smash/web/views/study.py b/smash/web/views/study.py
index 8d8bc07d..2c1cdd67 100644
--- a/smash/web/views/study.py
+++ b/smash/web/views/study.py
@@ -8,7 +8,10 @@ from web.decorators import PermissionDecorator
from web.forms import StudyColumnsEditForm, StudyEditForm, StudyNotificationParametersEditForm, \
StudyRedCapColumnsEditForm
from web.forms.custom_study_subject_field_forms import CustomStudySubjectFieldAddForm, CustomStudySubjectFieldEditForm
-from web.models import Study
+from web.forms.visit_import_data_form import VisitImportDataEditForm
+from web.importer import TnsCsvVisitImportReader
+from web.importer.log_storage import LogStorageHandler
+from web.models import Study, VisitImportData
from web.models.custom_data import CustomStudySubjectField
from web.views import wrap_response
@@ -51,11 +54,22 @@ def study_edit(request, study_id):
redcap_columns_form = StudyRedCapColumnsEditForm(instance=study.redcap_columns,
prefix="redcap")
+ etl_entries = []
+ for import_data in VisitImportData.objects.filter(study=study).all():
+ etl_entries.append({'type': 'Import visit',
+ 'file': import_data.filename,
+ 'filetype': 'CSV',
+ 'run_at': import_data.run_at_times,
+ 'id': import_data.id,
+ 'available': import_data.filename != '' and import_data.filename is not None,
+ 'worker': str(import_data.import_worker)
+ })
return wrap_response(request, 'study/edit.html', {
'study_form': study_form,
'notifications_form': notifications_form,
'study_columns_form': study_columns_form,
- 'redcap_columns_form': redcap_columns_form
+ 'redcap_columns_form': redcap_columns_form,
+ 'etl_entries': etl_entries
})
@@ -96,9 +110,52 @@ def custom_study_subject_field_edit(request, study_id, field_id):
})
+# noinspection PyUnusedLocal
@PermissionDecorator('change_study', 'configuration')
def custom_study_subject_field_delete(request, study_id, field_id):
study = get_object_or_404(Study, id=study_id)
field = get_object_or_404(CustomStudySubjectField, id=field_id)
field.delete()
return redirect('web.views.edit_study', study_id=study.id)
+
+
+@PermissionDecorator('change_study', 'configuration')
+def import_visit_edit(request, study_id, import_id):
+ study = get_object_or_404(Study, id=study_id)
+ import_data = get_object_or_404(VisitImportData, id=import_id)
+ if request.method == 'POST':
+ visit_import_data_form = VisitImportDataEditForm(request.POST, instance=import_data)
+ if visit_import_data_form.is_valid():
+ visit_import_data_form.save()
+ return redirect('web.views.edit_study', study_id=study.id)
+ else:
+ visit_import_data_form = VisitImportDataEditForm(instance=import_data)
+
+ return wrap_response(request, 'visit_import_data/edit.html', {
+ 'form': visit_import_data_form,
+ 'study_id': study.id
+ })
+
+
+@PermissionDecorator('change_study', 'configuration')
+def import_visit_execute(request, study_id, import_id):
+ study = get_object_or_404(Study, id=study_id)
+ import_data = get_object_or_404(VisitImportData, id=import_id)
+ if import_data.file_available():
+ reader = TnsCsvVisitImportReader(import_data)
+ log_storage = LogStorageHandler()
+ logging.getLogger('').addHandler(log_storage)
+ reader.load_data()
+ logging.getLogger('').removeHandler(log_storage)
+ messages.add_message(request, messages.SUCCESS,
+ str(reader.processed_count) + ' appointment(s) were added/updated successfully.')
+ if reader.problematic_count > 0:
+ messages.add_message(request, messages.ERROR,
+ str(reader.problematic_count) + ' problematic entries encountered.')
+ if "WARNING" in log_storage.level_messages:
+ for entry in log_storage.level_messages["WARNING"]:
+ messages.add_message(request, messages.WARNING, entry)
+ else:
+ messages.add_message(request, messages.ERROR, import_data.get_absolute_file_path() + ' is not available.')
+
+ return redirect('web.views.edit_study', study_id=study.id)
--
GitLab