form for configuring automatic subject import

smash/web/forms/subject_import_data_form.py

+from django import forms
+from django.forms import ModelForm
+
+from web.models import SubjectImportData, Subject, StudySubject
+
+
+class SubjectImportDataEditForm(ModelForm):
+    class Meta:
+        model = SubjectImportData
+        fields = '__all__'
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        instance = kwargs.get('instance')
+        if instance:
+            for field in Subject._meta.get_fields():
+                if field.get_internal_type() == "CharField" or \
+                        field.get_internal_type() == "DateField" or \
+                        field.get_internal_type() == "TextField":
+                    field_id = Subject._meta.db_table + " - " + field.name
+                    value = field.name
+                    for mapping in instance.column_mappings.all():
+                        if mapping.table_name == Subject._meta.db_table and field.name == mapping.column_name:
+                            value = mapping.csv_column_name
+                    self.fields[field_id] = forms.CharField(label=field.verbose_name + " column name ",
+                                                            initial=value)
+            for field in StudySubject._meta.get_fields():
+                if field.get_internal_type() == "CharField" or \
+                        field.get_internal_type() == "DateField" or \
+                        field.get_internal_type() == "TextField":
+                    field_id = StudySubject._meta.db_table + " - " + field.name
+                    value = field.name
+                    for mapping in instance.column_mappings.all():
+                        if mapping.table_name == StudySubject._meta.db_table and field.name == mapping.column_name:
+                            value = mapping.csv_column_name
+                    self.fields[field_id] = forms.CharField(label=field.verbose_name + " column name ",
+                                                            initial=value)
+
+    def save(self, commit=True) -> StudySubject:
+        instance = super().save(commit)
+        # we can add custom values only after object exists in the database
+        for field in Subject._meta.get_fields():
+            if field.get_internal_type() == "CharField" or \
+                    field.get_internal_type() == "DateField" or \
+                    field.get_internal_type() == "TextField":
+                field_id = Subject._meta.db_table + " - " + field.name
+                value = self[field_id].value()
+                instance.set_column_mapping(Subject, field.name, value)
+
+        for field in StudySubject._meta.get_fields():
+            if field.get_internal_type() == "CharField" or \
+                    field.get_internal_type() == "DateField" or \
+                    field.get_internal_type() == "TextField":
+                field_id = StudySubject._meta.db_table + " - " + field.name
+                value = self[field_id].value()
+                instance.set_column_mapping(StudySubject, field.name, value)
+        return instance

smash/web/importer/csv_subject_import_reader.py

@@ -53,10 +53,11 @@ class CsvSubjectImportReader(SubjectImportReader):
 
         if table == Subject:
             old_val = getattr(study_subject.subject, field.name)
-            setattr(study_subject.subject, field.name, self.get_new_value(old_val, column_name, value))
+            setattr(study_subject.subject, field.name, self.get_new_value(old_val, field.name, value))
         elif table == StudySubject:
             old_val = getattr(study_subject, field.name)
-            setattr(study_subject, field.name, self.get_new_value(old_val, column_name, value))
+            print(field.name + ": " + str(old_val) + " - " + str(value))
+            setattr(study_subject, field.name, self.get_new_value(old_val, field.name, value))
         else:
             logger.warning("Don't know how to handle column " + column_name + " with data " + value)
 

smash/web/importer/importer.py

@@ -56,8 +56,8 @@ class Importer(EtlCommon):
                 if field.get_internal_type() == "CharField" or \
                         field.get_internal_type() == "DateField" or \
                         field.get_internal_type() is "BooleanField":
-                    old_value = getattr(study_subject.subject, field.name)
-                    new_value = self.get_new_value(old_value, field.name, study_subject.subject)
+                    old_value = getattr(db_study_subject.subject, field.name)
+                    new_value = self.get_new_value(old_value, field.name, getattr(study_subject.subject, field.name))
                     self.create_provenance_and_change_data(db_study_subject.subject, field.name, new_value, Subject)
             db_study_subject.subject.save()
 
@@ -65,8 +65,8 @@ class Importer(EtlCommon):
                 if field.get_internal_type() == "CharField" or \
                         field.get_internal_type() == "DateField" or \
                         field.get_internal_type() is "BooleanField":
-                    old_value = getattr(study_subject, field.name)
-                    new_value = self.get_new_value(old_value, field.name, study_subject.subject)
+                    old_value = getattr(db_study_subject, field.name)
+                    new_value = self.get_new_value(old_value, field.name, getattr(study_subject, field.name))
                     self.create_provenance_and_change_data(db_study_subject, field.name, new_value, StudySubject)
             db_study_subject.save()
 

smash/web/models/etl/etl.py

 # coding=utf-8
 import logging
 import os
+from typing import Type
 
 from django.conf import settings
 from django.db import models
 
+from .etl_column_mapping import EtlColumnMapping
+
 logger = logging.getLogger(__name__)
 
 
@@ -60,3 +63,13 @@ class EtlData(models.Model):
 
     def get_absolute_file_path(self) -> str:
         return os.path.join(settings.ETL_ROOT, self.filename)
+
+    def set_column_mapping(self, object_type: Type[models.Model], column_name: str, csv_column_name: str):
+        for entry in self.column_mappings.all():
+            if entry.table_name == object_type._meta.db_table and \
+                    entry.column_name == column_name:
+                entry.csv_column_name = csv_column_name
+                entry.save()
+                return
+        EtlColumnMapping.objects.create(etl_data=self, table_name=object_type._meta.db_table, column_name=column_name,
+                                        csv_column_name=csv_column_name)

smash/web/templates/study/edit.html

@@ -121,6 +121,8 @@
                                         <td class="text-center"><a title="Edit ETL"
                                                 {% if etl_entry.type == 'Import visit' %}
                                                                    href="{% url 'web.views.import_visit_edit' study_id etl_entry.id %}"
+                                                {% elif etl_entry.type == 'Subject visit' %}
+                                                                   href="{% url 'web.views.import_subject_edit' study_id etl_entry.id %}"
                                                 {% else %}
                                                                    href="#"
                                                 {% endif %}
@@ -129,9 +131,16 @@
                                         </td>
                                         <td class="text-center">
                                             {% if etl_entry.available %}
-                                                <a href="{% url 'web.views.import_visit_execute' study_id etl_entry.id %}"
-                                                   type="button"
-                                                   class="btn btn-block btn-default">Run</a>
+                                                <a
+                                                {% if etl_entry.type == 'Import visit' %}
+                                                    href="{% url 'web.views.import_visit_execute' study_id etl_entry.id %}"
+                                                {% elif etl_entry.type == 'Subject visit' %}
+                                                    href="{% url 'web.views.import_subject_execute' study_id etl_entry.id %}"
+                                                {% else %}
+                                                    href="#"
+                                                {% endif %}
+                                                type="button"
+                                                class="btn btn-block btn-default">Run</a>
                                             {% else %}
                                                 <a type="button"
                                                    class="btn btn-block btn-warning" disabled>Unavailable</a>

smash/web/templates/subject_import_data/edit.html

+{% extends "_base.html" %}
+{% load static %}
+{% load filters %}
+
+{% block styles %}
+    {{ block.super }}
+    <link rel="stylesheet" href="{% static 'AdminLTE/plugins/awesomplete/awesomplete.css' %}"/>
+
+    {% include "includes/datepicker.css.html" %}
+{% endblock styles %}
+
+{% block ui_active_tab %}'subjects'{% endblock ui_active_tab %}
+{% block page_header %}Edit subject import data{% endblock page_header %}
+{% block page_description %}{% endblock page_description %}
+
+{% block title %}{{ block.super }} - Edit subject import data{% endblock %}
+
+{% block breadcrumb %}
+    {% include "subjects/breadcrumb.html" %}
+{% endblock breadcrumb %}
+
+{% block maincontent %}
+
+    {% block content %}
+        <div class="row">
+            <div class="col-md-12">
+                <div class="box box-success">
+                    <div class="box-header with-border">
+                        <h3 class="box-title">Enter subject import details</h3>
+                    </div>
+
+
+                    <form method="post" action="" class="form-horizontal">
+                        {% csrf_token %}
+
+                        <div class="box-body">
+                            {% for field in form %}
+                                <div class="form-group {% if field.errors %}has-error{% endif %}">
+                                    <label class="col-sm-4  col-lg-offset-1 col-lg-2 control-label">
+                                        {{ field.label }}
+                                    </label>
+
+                                    <div class="col-sm-8 col-lg-4">
+                                        {{ field|add_class:'form-control' }}
+                                    </div>
+
+                                    {% if field.errors %}
+                                        <span class="help-block">
+  							{{ field.errors }}
+  						</span>
+                                    {% endif %}
+                                </div>
+                            {% endfor %}
+                        </div><!-- /.box-body -->
+                        <div class="box-footer">
+                            <div class="col-sm-6">
+                                <button type="submit" class="btn btn-block btn-success">Save</button>
+                            </div>
+                            <div class="col-sm-6">
+                                <a href="{% url 'web.views.edit_study' study_id %}"
+                                   class="btn btn-block btn-default">Cancel</a>
+                            </div>
+                        </div><!-- /.box-footer -->
+                    </form>
+                </div>
+
+            </div>
+        </div>
+
+    {% endblock %}
+
+
+{% endblock maincontent %}
+
+{% block scripts %}
+    {{ block.super }}
+
+    <script src="{% static 'AdminLTE/plugins/awesomplete/awesomplete.min.js' %}"></script>
+
+    {% include "includes/datetimepicker.js.html" %}
+{% endblock scripts %}

smash/web/urls.py

@@ -254,9 +254,14 @@ urlpatterns = [
         views.study.custom_study_subject_field_delete, name='web.views.custom_study_subject_field_delete'),
 
     url(r'^study/(?P<study_id>\d+)/import_visit_edit/(?P<import_id>\d+)/edit',
-        views.study.import_visit_edit, name='web.views.import_visit_edit'),
+        views.etl.import_visit_edit, name='web.views.import_visit_edit'),
     url(r'^study/(?P<study_id>\d+)/import_visit_edit/(?P<import_id>\d+)/execute',
-        views.study.import_visit_execute, name='web.views.import_visit_execute'),
+        views.etl.import_visit_execute, name='web.views.import_visit_execute'),
+
+    url(r'^study/(?P<study_id>\d+)/import_subject_edit/(?P<import_id>\d+)/edit',
+        views.etl.import_subject_edit, name='web.views.import_subject_edit'),
+    url(r'^study/(?P<study_id>\d+)/import_subject_edit/(?P<import_id>\d+)/execute',
+        views.etl.import_subject_execute, name='web.views.import_subject_execute'),
 
     ####################
     #       EXPORT     #

smash/web/views/__init__.py

@@ -103,6 +103,7 @@ from . import redcap
 from . import rooms
 from . import uploaded_files
 from . import study
+from . import etl
 from . import password
 from . import appointment_type
 from . import provenance
\ No newline at end of file

smash/web/views/etl.py

+# coding=utf-8
+import logging
+
+from django.contrib import messages
+from django.shortcuts import redirect, get_object_or_404
+
+from web.decorators import PermissionDecorator
+from web.forms.subject_import_data_form import SubjectImportDataEditForm
+from web.forms.visit_import_data_form import VisitImportDataEditForm
+from web.importer import Importer, CsvSubjectImportReader, TnsCsvVisitImportReader
+from web.importer.log_storage import LogStorageHandler
+from web.models import Study, VisitImportData, SubjectImportData
+from web.views import wrap_response
+
+logger = logging.getLogger(__name__)
+
+
+@PermissionDecorator('change_study', 'configuration')
+def import_visit_edit(request, study_id, import_id):
+    study = get_object_or_404(Study, id=study_id)
+    import_data = get_object_or_404(VisitImportData, id=import_id)
+    if request.method == 'POST':
+        visit_import_data_form = VisitImportDataEditForm(request.POST, instance=import_data)
+        if visit_import_data_form.is_valid():
+            visit_import_data_form.save()
+            return redirect('web.views.edit_study', study_id=study.id)
+    else:
+        visit_import_data_form = VisitImportDataEditForm(instance=import_data)
+
+    return wrap_response(request, 'visit_import_data/edit.html', {
+        'form': visit_import_data_form,
+        'study_id': study.id
+    })
+
+
+@PermissionDecorator('change_study', 'configuration')
+def import_visit_execute(request, study_id, import_id):
+    study = get_object_or_404(Study, id=study_id)
+    import_data = get_object_or_404(VisitImportData, id=import_id)
+    if import_data.file_available():
+        reader = TnsCsvVisitImportReader(import_data)
+        log_storage = LogStorageHandler()
+        logging.getLogger('').addHandler(log_storage)
+        reader.load_data()
+        logging.getLogger('').removeHandler(log_storage)
+        messages.add_message(request, messages.SUCCESS,
+                             str(reader.processed_count) + ' appointment(s) were added/updated successfully.')
+        if reader.problematic_count > 0:
+            messages.add_message(request, messages.ERROR,
+                                 str(reader.problematic_count) + ' problematic entries encountered.')
+        if "WARNING" in log_storage.level_messages:
+            for entry in log_storage.level_messages["WARNING"]:
+                messages.add_message(request, messages.WARNING, entry)
+    else:
+        messages.add_message(request, messages.ERROR, import_data.get_absolute_file_path() + ' is not available.')
+
+    return redirect('web.views.edit_study', study_id=study.id)
+
+
+@PermissionDecorator('change_study', 'configuration')
+def import_subject_edit(request, study_id, import_id):
+    study = get_object_or_404(Study, id=study_id)
+    import_data = get_object_or_404(SubjectImportData, id=import_id)
+    if request.method == 'POST':
+        subject_import_data_form = SubjectImportDataEditForm(request.POST, instance=import_data)
+        if subject_import_data_form.is_valid():
+            subject_import_data_form.save()
+            return redirect('web.views.edit_study', study_id=study.id)
+    else:
+        subject_import_data_form = SubjectImportDataEditForm(instance=import_data)
+
+    return wrap_response(request, 'subject_import_data/edit.html', {
+        'form': subject_import_data_form,
+        'study_id': study.id
+    })
+
+
+@PermissionDecorator('change_study', 'configuration')
+def import_subject_execute(request, study_id, import_id):
+    study = get_object_or_404(Study, id=study_id)
+    import_data = get_object_or_404(SubjectImportData, id=import_id)
+    if import_data.file_available():
+        reader = Importer(CsvSubjectImportReader(import_data))
+        log_storage = LogStorageHandler()
+        logging.getLogger('').addHandler(log_storage)
+        reader.execute()
+        logging.getLogger('').removeHandler(log_storage)
+        if reader.added_count > 0:
+            messages.add_message(request, messages.SUCCESS,
+                                 str(reader.added_count) + ' subject(s) were added successfully.')
+        if reader.merged_count > 0:
+            messages.add_message(request, messages.SUCCESS,
+                                 str(reader.merged_count) + ' subject(s) were updated successfully.')
+        if reader.problematic_count > 0:
+            messages.add_message(request, messages.ERROR,
+                                 str(reader.problematic_count) + ' problematic entries encountered.')
+        if "WARNING" in log_storage.level_messages:
+            for entry in log_storage.level_messages["WARNING"]:
+                messages.add_message(request, messages.WARNING, entry)
+    else:
+        messages.add_message(request, messages.ERROR, import_data.get_absolute_file_path() + ' is not available.')
+
+    return redirect('web.views.edit_study', study_id=study.id)

smash/web/views/study.py

@@ -8,10 +8,7 @@ from web.decorators import PermissionDecorator
 from web.forms import StudyColumnsEditForm, StudyEditForm, StudyNotificationParametersEditForm, \
     StudyRedCapColumnsEditForm
 from web.forms.custom_study_subject_field_forms import CustomStudySubjectFieldAddForm, CustomStudySubjectFieldEditForm
-from web.forms.visit_import_data_form import VisitImportDataEditForm
-from web.importer import TnsCsvVisitImportReader
-from web.importer.log_storage import LogStorageHandler
-from web.models import Study, VisitImportData
+from web.models import Study, VisitImportData, SubjectImportData
 from web.models.custom_data import CustomStudySubjectField
 from web.views import wrap_response
 
@@ -64,6 +61,15 @@ def study_edit(request, study_id):
                             'available': import_data.filename != '' and import_data.filename is not None,
                             'worker': str(import_data.import_worker)
                             })
+    for import_data in SubjectImportData.objects.filter(study=study).all():
+        etl_entries.append({'type': 'Subject visit',
+                            'file': import_data.filename,
+                            'filetype': 'CSV',
+                            'run_at': import_data.run_at_times,
+                            'id': import_data.id,
+                            'available': import_data.filename != '' and import_data.filename is not None,
+                            'worker': str(import_data.import_worker)
+                            })
     return wrap_response(request, 'study/edit.html', {
         'study_form': study_form,
         'notifications_form': notifications_form,
@@ -117,45 +123,3 @@ def custom_study_subject_field_delete(request, study_id, field_id):
     field = get_object_or_404(CustomStudySubjectField, id=field_id)
     field.delete()
     return redirect('web.views.edit_study', study_id=study.id)
-
-
-@PermissionDecorator('change_study', 'configuration')
-def import_visit_edit(request, study_id, import_id):
-    study = get_object_or_404(Study, id=study_id)
-    import_data = get_object_or_404(VisitImportData, id=import_id)
-    if request.method == 'POST':
-        visit_import_data_form = VisitImportDataEditForm(request.POST, instance=import_data)
-        if visit_import_data_form.is_valid():
-            visit_import_data_form.save()
-            return redirect('web.views.edit_study', study_id=study.id)
-    else:
-        visit_import_data_form = VisitImportDataEditForm(instance=import_data)
-
-    return wrap_response(request, 'visit_import_data/edit.html', {
-        'form': visit_import_data_form,
-        'study_id': study.id
-    })
-
-
-@PermissionDecorator('change_study', 'configuration')
-def import_visit_execute(request, study_id, import_id):
-    study = get_object_or_404(Study, id=study_id)
-    import_data = get_object_or_404(VisitImportData, id=import_id)
-    if import_data.file_available():
-        reader = TnsCsvVisitImportReader(import_data)
-        log_storage = LogStorageHandler()
-        logging.getLogger('').addHandler(log_storage)
-        reader.load_data()
-        logging.getLogger('').removeHandler(log_storage)
-        messages.add_message(request, messages.SUCCESS,
-                             str(reader.processed_count) + ' appointment(s) were added/updated successfully.')
-        if reader.problematic_count > 0:
-            messages.add_message(request, messages.ERROR,
-                                 str(reader.problematic_count) + ' problematic entries encountered.')
-        if "WARNING" in log_storage.level_messages:
-            for entry in log_storage.level_messages["WARNING"]:
-                messages.add_message(request, messages.WARNING, entry)
-    else:
-        messages.add_message(request, messages.ERROR, import_data.get_absolute_file_path() + ' is not available.')
-
-    return redirect('web.views.edit_study', study_id=study.id)