form that allows editing visit import data

smash/web/forms/visit_import_data_form.py

+from django.forms import ModelForm
+
+from web.models import VisitImportData
+
+
+class VisitImportDataEditForm(ModelForm):
+    class Meta:
+        model = VisitImportData
+        fields = '__all__'
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)

smash/web/importer/csv_tns_visit_import_reader.py

@@ -194,6 +194,11 @@ class TnsCsvVisitImportReader:
         try:
             visit_number = data[self.visit_import_data.visit_number_column_name]
             visit_number = int(visit_number) + (1 - self.visit_import_data.study.redcap_first_visit_number)
+            if visit_number < 1:
+                logger.warning(
+                    "Visit number is invalid. Visit number should start from: " +
+                    str(self.visit_import_data.study.redcap_first_visit_number) + ".")
+                visit_number = 1
             return visit_number
         except KeyError as e:
             raise EtlException('Visit number is not defined') from e

smash/web/importer/log_storage.py

+import logging
+
+
+class LogStorageHandler(logging.Handler):
+    level_messages = {}
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.level_messages = {}
+
+    def emit(self, record):
+        level = record.levelname
+        if level not in self.level_messages:
+            self.level_messages[level] = []
+        self.level_messages[level].append(self.format(record))

smash/web/importer/warning_counter.py

 import logging
 
+
 class MsgCounterHandler(logging.Handler):
     level2count = None
 
     def __init__(self, *args, **kwargs):
-        super(MsgCounterHandler, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self.level2count = {}
 
     def emit(self, record):
-        l = record.levelname
-        if l not in self.level2count:
-            self.level2count[l] = 0
-        self.level2count[l] += 1
+        level = record.levelname
+        if level not in self.level2count:
+            self.level2count[level] = 0
+        self.level2count[level] += 1

smash/web/templates/study/edit.html

@@ -94,6 +94,56 @@
                             </div>
                         </div><!-- /.box-body -->
 
+                        <div class="box-header with-border">
+                            <h3>ETL</h3>
+                        </div>
+                        <div class="box-body">
+                            <table class="table table-bordered table-striped">
+                                <thead>
+                                <tr>
+                                    <th class="text-center">Action Type</th>
+                                    <th class="text-center">File</th>
+                                    <th class="text-center">File Type</th>
+                                    <th class="text-center">Run at</th>
+                                    <th class="text-center">Worker</th>
+                                    <th class="text-center">Edit</th>
+                                    <th class="text-center">Run manually</th>
+                                </tr>
+                                </thead>
+                                <tbody>
+                                {% for etl_entry in etl_entries %}
+                                    <tr>
+                                        <td class="text-center">{{ etl_entry.type }}</td>
+                                        <td class="text-center">{{ etl_entry.file }}</td>
+                                        <td class="text-center">{{ etl_entry.filetype }}</td>
+                                        <td class="text-center">{{ etl_entry.run_at }}</td>
+                                        <td class="text-center">{{ etl_entry.worker }}</td>
+                                        <td class="text-center"><a title="Edit ETL"
+                                                {% if etl_entry.type == 'Import visit' %}
+                                                                   href="{% url 'web.views.import_visit_edit' study_id etl_entry.id %}"
+                                                {% else %}
+                                                                   href="#"
+                                                {% endif %}
+                                                                   type="button"
+                                                                   class="btn btn-block btn-default">EDIT</a>
+                                        </td>
+                                        <td class="text-center">
+                                            {% if etl_entry.available %}
+                                                <a href="{% url 'web.views.import_visit_execute' study_id etl_entry.id %}"
+                                                   type="button"
+                                                   class="btn btn-block btn-default">Run</a>
+                                            {% else %}
+                                                <a type="button"
+                                                   class="btn btn-block btn-warning" disabled>Unavailable</a>
+
+                                            {% endif %}
+                                        </td>
+                                    </tr>
+                                {% endfor %}
+                                </tbody>
+                            </table>
+                        </div>
+
                         {% include 'includes/custom_study_subject_field_box.html' with study=study fields=study.customstudysubjectfield_set.all %}
 
                         <div class="box-header with-border">

smash/web/templates/visit_import_data/edit.html

+{% extends "_base.html" %}
+{% load static %}
+{% load filters %}
+
+{% block styles %}
+    {{ block.super }}
+    <link rel="stylesheet" href="{% static 'AdminLTE/plugins/awesomplete/awesomplete.css' %}"/>
+
+    {% include "includes/datepicker.css.html" %}
+{% endblock styles %}
+
+{% block ui_active_tab %}'subjects'{% endblock ui_active_tab %}
+{% block page_header %}Edit visit import data{% endblock page_header %}
+{% block page_description %}{% endblock page_description %}
+
+{% block title %}{{ block.super }} - Edit visit import data{% endblock %}
+
+{% block breadcrumb %}
+    {% include "subjects/breadcrumb.html" %}
+{% endblock breadcrumb %}
+
+{% block maincontent %}
+
+    {% block content %}
+        <div class="row">
+            <div class="col-md-12">
+                <div class="box box-success">
+                    <div class="box-header with-border">
+                        <h3 class="box-title">Enter visit import details</h3>
+                    </div>
+
+
+                    <form method="post" action="" class="form-horizontal">
+                        {% csrf_token %}
+
+                        <div class="box-body">
+                            {% for field in form %}
+                                <div class="form-group {% if field.errors %}has-error{% endif %}">
+                                    <label class="col-sm-4  col-lg-offset-1 col-lg-2 control-label">
+                                        {{ field.label }}
+                                    </label>
+
+                                    <div class="col-sm-8 col-lg-4">
+                                        {{ field|add_class:'form-control' }}
+                                    </div>
+
+                                    {% if field.errors %}
+                                        <span class="help-block">
+  							{{ field.errors }}
+  						</span>
+                                    {% endif %}
+                                </div>
+                            {% endfor %}
+                        </div><!-- /.box-body -->
+                        <div class="box-footer">
+                            <div class="col-sm-6">
+                                <button type="submit" class="btn btn-block btn-success">Save</button>
+                            </div>
+                            <div class="col-sm-6">
+                                <a href="{% url 'web.views.edit_study' study_id %}"
+                                   class="btn btn-block btn-default">Cancel</a>
+                            </div>
+                        </div><!-- /.box-footer -->
+                    </form>
+                </div>
+
+            </div>
+        </div>
+
+    {% endblock %}
+
+
+{% endblock maincontent %}
+
+{% block scripts %}
+    {{ block.super }}
+
+    <script src="{% static 'AdminLTE/plugins/awesomplete/awesomplete.min.js' %}"></script>
+
+    {% include "includes/datetimepicker.js.html" %}
+{% endblock scripts %}

smash/web/urls.py

@@ -253,6 +253,11 @@ urlpatterns = [
     url(r'^study/(?P<study_id>\d+)/custom_study_subject_field/(?P<field_id>\d+)/delete',
         views.study.custom_study_subject_field_delete, name='web.views.custom_study_subject_field_delete'),
 
+    url(r'^study/(?P<study_id>\d+)/import_visit_edit/(?P<import_id>\d+)/edit',
+        views.study.import_visit_edit, name='web.views.import_visit_edit'),
+    url(r'^study/(?P<study_id>\d+)/import_visit_edit/(?P<import_id>\d+)/execute',
+        views.study.import_visit_execute, name='web.views.import_visit_execute'),
+
     ####################
     #       EXPORT     #
     ####################

smash/web/views/study.py

@@ -8,7 +8,10 @@ from web.decorators import PermissionDecorator
 from web.forms import StudyColumnsEditForm, StudyEditForm, StudyNotificationParametersEditForm, \
     StudyRedCapColumnsEditForm
 from web.forms.custom_study_subject_field_forms import CustomStudySubjectFieldAddForm, CustomStudySubjectFieldEditForm
-from web.models import Study
+from web.forms.visit_import_data_form import VisitImportDataEditForm
+from web.importer import TnsCsvVisitImportReader
+from web.importer.log_storage import LogStorageHandler
+from web.models import Study, VisitImportData
 from web.models.custom_data import CustomStudySubjectField
 from web.views import wrap_response
 
@@ -51,11 +54,22 @@ def study_edit(request, study_id):
         redcap_columns_form = StudyRedCapColumnsEditForm(instance=study.redcap_columns,
                                                          prefix="redcap")
 
+    etl_entries = []
+    for import_data in VisitImportData.objects.filter(study=study).all():
+        etl_entries.append({'type': 'Import visit',
+                            'file': import_data.filename,
+                            'filetype': 'CSV',
+                            'run_at': import_data.run_at_times,
+                            'id': import_data.id,
+                            'available': import_data.filename != '' and import_data.filename is not None,
+                            'worker': str(import_data.import_worker)
+                            })
     return wrap_response(request, 'study/edit.html', {
         'study_form': study_form,
         'notifications_form': notifications_form,
         'study_columns_form': study_columns_form,
-        'redcap_columns_form': redcap_columns_form
+        'redcap_columns_form': redcap_columns_form,
+        'etl_entries': etl_entries
     })
 
 
@@ -96,9 +110,52 @@ def custom_study_subject_field_edit(request, study_id, field_id):
     })
 
 
+# noinspection PyUnusedLocal
 @PermissionDecorator('change_study', 'configuration')
 def custom_study_subject_field_delete(request, study_id, field_id):
     study = get_object_or_404(Study, id=study_id)
     field = get_object_or_404(CustomStudySubjectField, id=field_id)
     field.delete()
     return redirect('web.views.edit_study', study_id=study.id)
+
+
+@PermissionDecorator('change_study', 'configuration')
+def import_visit_edit(request, study_id, import_id):
+    study = get_object_or_404(Study, id=study_id)
+    import_data = get_object_or_404(VisitImportData, id=import_id)
+    if request.method == 'POST':
+        visit_import_data_form = VisitImportDataEditForm(request.POST, instance=import_data)
+        if visit_import_data_form.is_valid():
+            visit_import_data_form.save()
+            return redirect('web.views.edit_study', study_id=study.id)
+    else:
+        visit_import_data_form = VisitImportDataEditForm(instance=import_data)
+
+    return wrap_response(request, 'visit_import_data/edit.html', {
+        'form': visit_import_data_form,
+        'study_id': study.id
+    })
+
+
+@PermissionDecorator('change_study', 'configuration')
+def import_visit_execute(request, study_id, import_id):
+    study = get_object_or_404(Study, id=study_id)
+    import_data = get_object_or_404(VisitImportData, id=import_id)
+    if import_data.file_available():
+        reader = TnsCsvVisitImportReader(import_data)
+        log_storage = LogStorageHandler()
+        logging.getLogger('').addHandler(log_storage)
+        reader.load_data()
+        logging.getLogger('').removeHandler(log_storage)
+        messages.add_message(request, messages.SUCCESS,
+                             str(reader.processed_count) + ' appointment(s) were added/updated successfully.')
+        if reader.problematic_count > 0:
+            messages.add_message(request, messages.ERROR,
+                                 str(reader.problematic_count) + ' problematic entries encountered.')
+        if "WARNING" in log_storage.level_messages:
+            for entry in log_storage.level_messages["WARNING"]:
+                messages.add_message(request, messages.WARNING, entry)
+    else:
+        messages.add_message(request, messages.ERROR, import_data.get_absolute_file_path() + ' is not available.')
+
+    return redirect('web.views.edit_study', study_id=study.id)