visit import data moved to separate data structure

smash/web/importer/csv_tns_visit_import_reader.py

@@ -8,9 +8,9 @@ import traceback
 
 import pytz
 
-from web.models import StudySubject, Study, Visit, Appointment, AppointmentType, Location, AppointmentTypeLink, \
-    Subject, User, Worker, Provenance, ConfigurationItem
-from web.models.constants import IMPORTER_USER, IMPORT_APPOINTMENT_TYPE
+from web.models import StudySubject, Visit, Appointment, Location, AppointmentTypeLink, \
+    Subject, Provenance
+from web.models.etl.visit_import import VisitImportData
 from .warning_counter import MsgCounterHandler
 
 CSV_DATE_FORMAT = "%d/%m/%Y"
@@ -19,29 +19,19 @@ logger = logging.getLogger(__name__)
 
 
 class TnsCsvVisitImportReader:
-    def __init__(self, study_id: int):
-        self.study = Study.objects.get(pk=study_id)
-        appointment_code = ConfigurationItem.objects.get(type=IMPORT_APPOINTMENT_TYPE).value
+    def __init__(self, data: VisitImportData):
+        self.visit_import_data = data
+
+        self.appointment_type = data.appointment_type
+        if data.appointment_type is None:
+            logger.warning("Appointment is not assigned")
 
-        appointment_types = AppointmentType.objects.filter(code=appointment_code)
-        if len(appointment_types) > 0:
-            self.appointment_type = appointment_types[0]
-        else:
-            logger.warning("Appointment type does not exist: " + appointment_code)
-            self.appointment_type = None
         self.problematic_count = 0
         self.processed_count = 0
         self.warning_count = 0
 
-        self.importer_user = None
-
-        importer_user_name = ConfigurationItem.objects.get(type=IMPORTER_USER).value
-        if importer_user_name is not None and importer_user_name != '':
-            user = User.objects.filter(username=importer_user_name)
-            if user is None:
-                logger.warning("User does not exist: " + importer_user_name)
-            else:
-                self.importer_user = Worker.objects.filter(user=user)
+        if data.import_worker is None:
+            logger.warning("Import user is not assigned")
 
     def load_data(self, filename):
         warning_counter = MsgCounterHandler()
@@ -62,7 +52,7 @@ class TnsCsvVisitImportReader:
                         logger.debug("Subject " + nd_number + " does not exist. Creating")
                         subject = Subject.objects.create()
                         study_subject = StudySubject.objects.create(subject=subject,
-                                                                    study=self.study,
+                                                                    study=self.visit_import_data.study,
                                                                     nd_number=nd_number,
                                                                     screening_number=nd_number)
                     else:
@@ -101,7 +91,7 @@ class TnsCsvVisitImportReader:
                             description = '{} changed from "{}" to "{}"'.format(field, old_value, new_value)
                             p = Provenance(modified_table=Visit._meta.db_table,
                                            modified_table_id=visit.id,
-                                           modification_author=self.importer_user,
+                                           modification_author=self.visit_import_data.import_worker,
                                            previous_value=old_value,
                                            new_value=new_value,
                                            modification_description=description,
@@ -123,7 +113,7 @@ class TnsCsvVisitImportReader:
                                     description = '{} changed from "{}" to "{}"'.format(field, '', new_value)
                                     p = Provenance(modified_table=Visit._meta.db_table,
                                                    modified_table_id=visit.id,
-                                                   modification_author=self.importer_user,
+                                                   modification_author=self.visit_import_data.import_worker,
                                                    previous_value='',
                                                    new_value=new_value,
                                                    modification_description=description,
@@ -150,7 +140,7 @@ class TnsCsvVisitImportReader:
                             description = '{} changed from "{}" to "{}"'.format(field, old_value, new_value)
                             p = Provenance(modified_table=Appointment._meta.db_table,
                                            modified_table_id=appointment.id,
-                                           modification_author=self.importer_user,
+                                           modification_author=self.visit_import_data.import_worker,
                                            previous_value=old_value,
                                            new_value=new_value,
                                            modification_description=description,
@@ -175,7 +165,7 @@ class TnsCsvVisitImportReader:
                                     description = '{} changed from "{}" to "{}"'.format(field, '', new_value)
                                     p = Provenance(modified_table=Appointment._meta.db_table,
                                                    modified_table_id=appointment.id,
-                                                   modification_author=self.importer_user,
+                                                   modification_author=self.visit_import_data.import_worker,
                                                    previous_value='',
                                                    new_value=new_value,
                                                    modification_description=description,

smash/web/migrations/0179_visitimportdata.py

+# Generated by Django 2.0.13 on 2020-11-17 07:22
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('web', '0178_auto_20201116_1250'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='VisitImportData',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('appointment_type', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='web.AppointmentType', verbose_name='Default appointment type')),
+                ('import_worker', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='web.Worker', verbose_name='Worker used by importer')),
+                ('study', models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, to='web.Study', verbose_name='Study')),
+            ],
+        ),
+    ]

smash/web/migrations/0180_visitimportdata_migration.py

+from django.db import migrations
+
+from web.models.constants import IMPORTER_USER, GLOBAL_STUDY_ID, IMPORT_APPOINTMENT_TYPE
+
+
+def get_val(apps, item_type: str):
+    # noinspection PyPep8Naming
+    ConfigurationItem = apps.get_model("web", "ConfigurationItem")
+    items = ConfigurationItem.objects.filter(type=item_type)
+    if len(items) == 0:
+        return None
+    return items[0].value
+
+
+# noinspection PyUnusedLocal
+def create_visit_import_data(apps, schema_editor):
+    # noinspection PyPep8Naming
+    VisitImportData = apps.get_model("web", "VisitImportData")
+    # noinspection PyPep8Naming
+    Study = apps.get_model("web", "Study")
+    # noinspection PyPep8Naming
+    Worker = apps.get_model("web", "Worker")
+    # noinspection PyPep8Naming
+    AppointmentType = apps.get_model("web", "AppointmentType")
+
+    entry = VisitImportData()
+    entry.study = Study.objects.get(pk=GLOBAL_STUDY_ID)
+    importer_user_name = get_val(apps, IMPORTER_USER)
+    appointment_type_name = get_val(apps, IMPORT_APPOINTMENT_TYPE)
+
+    if importer_user_name is not None:
+        workers = Worker.objects.filter(user__username=importer_user_name)
+
+        if len(workers) > 0:
+            entry.worker = workers[0]
+
+    if appointment_type_name is not None:
+        appointment_type = AppointmentType.objects.filter(code=appointment_type_name)
+
+        if len(appointment_type) > 0:
+            entry.appointment_type = appointment_type[0]
+    entry.save()
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('web', '0179_visitimportdata'),
+    ]
+
+    operations = [
+        migrations.RunPython(create_visit_import_data),
+    ]

smash/web/models/__init__.py

@@ -39,8 +39,11 @@ from .mail_template import MailTemplate
 from .missing_subject import MissingSubject
 from .inconsistent_subject import InconsistentSubject, InconsistentField
 
+from .etl import VisitImportData
+
 __all__ = [Study, FlyingTeam, Appointment, AppointmentType, Availability, Holiday, Item, Language, Location, Room,
            Subject, StudySubject, StudySubjectList, SubjectColumns, StudyNotificationParameters,
            AppointmentList, AppointmentColumns, Visit, Worker, ContactAttempt, ConfigurationItem, MailTemplate,
            AppointmentTypeLink, VoucherType, VoucherTypePrice, Voucher, WorkerStudyRole,
-           MissingSubject, InconsistentSubject, InconsistentField, Country, StudyColumns, StudyRedCapColumns, VisitColumns, StudyVisitList]
+           MissingSubject, InconsistentSubject, InconsistentField, Country, StudyColumns, StudyRedCapColumns,
+           VisitColumns, StudyVisitList]

smash/web/models/etl/__init__.py

+from .visit_import import VisitImportData
+
+__all__ = [VisitImportData]

smash/web/models/etl/visit_import.py

+# coding=utf-8
+
+from django.db import models
+
+from web.models.constants import CUSTOM_FIELD_TYPE
+
+
+class VisitImportData(models.Model):
+    study = models.ForeignKey("web.Study",
+                              verbose_name='Study',
+                              editable=False,
+                              null=False,
+                              on_delete=models.CASCADE
+                              )
+
+    appointment_type = models.ForeignKey("web.AppointmentType",
+                                         verbose_name='Default appointment type',
+                                         blank=True,
+                                         null=True,
+                                         on_delete=models.CASCADE
+                                         )
+
+    import_worker = models.ForeignKey("web.Worker",
+                                      verbose_name='Worker used by importer',
+                                      blank=True,
+                                      null=True,
+                                      on_delete=models.CASCADE
+                                      )

smash/web/tests/importer/test_tns_csv_visit_import_reader.py

@@ -6,25 +6,22 @@ from django.test import TestCase
 from django.utils import timezone
 
 from web.importer import TnsCsvVisitImportReader, MsgCounterHandler
-from web.models import Appointment, Visit, StudySubject, AppointmentTypeLink, AppointmentType, ConfigurationItem
-from web.models.constants import IMPORT_APPOINTMENT_TYPE
+from web.models import Appointment, Visit, StudySubject, AppointmentTypeLink, AppointmentType, VisitImportData
 from web.tests.functions import get_resource_path, create_study_subject, create_appointment_type, create_location, \
-    get_test_study
+    get_test_study, create_worker
 
 logger = logging.getLogger(__name__)
 
 
 class TestTnsCsvVisitReader(TestCase):
     def setUp(self):
-        self.study = get_test_study()
+        appointment_type = create_appointment_type(code="SAMPLE_2")
+        self.visit_import_data = VisitImportData.objects.create(study=get_test_study(),
+                                                                appointment_type=appointment_type,
+                                                                import_worker=create_worker())
         self.warning_counter = MsgCounterHandler()
         logging.getLogger('').addHandler(self.warning_counter)
 
-        item = ConfigurationItem.objects.get(type=IMPORT_APPOINTMENT_TYPE)
-        item.value = "SAMPLE_2"
-        item.save()
-        create_appointment_type(code="SAMPLE_2")
-
         create_study_subject(nd_number='cov-000111')
         create_study_subject(nd_number='cov-222333')
         create_study_subject(nd_number='cov-444444')
@@ -38,7 +35,7 @@ class TestTnsCsvVisitReader(TestCase):
 
     def test_load_data(self):
         filename = get_resource_path('tns_vouchers_import.csv')
-        visits = TnsCsvVisitImportReader(self.study.id).load_data(filename)
+        visits = TnsCsvVisitImportReader(self.visit_import_data).load_data(filename)
         self.assertEqual(3, len(visits))
         visit = Visit.objects.filter(id=visits[0].id)[0]
         self.assertEqual("cov-000111", visit.subject.nd_number)
@@ -60,7 +57,7 @@ class TestTnsCsvVisitReader(TestCase):
                                      datetime_end=timezone.now(),
                                      datetime_begin=timezone.now(),
                                      visit_number=1)
-        visits = TnsCsvVisitImportReader(self.study.id).load_data(filename)
+        visits = TnsCsvVisitImportReader(self.visit_import_data).load_data(filename)
         visit = Visit.objects.filter(id=visits[0].id)[0]
         self.assertEqual("cov-000111", visit.subject.nd_number)
 
@@ -86,7 +83,7 @@ class TestTnsCsvVisitReader(TestCase):
         AppointmentTypeLink.objects.create(appointment_id=appointment.id,
                                            appointment_type=AppointmentType.objects.filter(code="SAMPLE_2")[0])
 
-        visits = TnsCsvVisitImportReader(self.study.id).load_data(filename)
+        visits = TnsCsvVisitImportReader(self.visit_import_data).load_data(filename)
         visit = Visit.objects.filter(id=visits[0].id)[0]
         self.assertEqual("cov-000111", visit.subject.nd_number)
 
@@ -105,7 +102,7 @@ class TestTnsCsvVisitReader(TestCase):
     def test_load_data_with_visit_and_no_previous_visits(self):
         filename = get_resource_path('tns_vouchers_3_import.csv')
 
-        visits = TnsCsvVisitImportReader(self.study.id).load_data(filename)
+        visits = TnsCsvVisitImportReader(self.visit_import_data).load_data(filename)
 
         subject_visits = Visit.objects.filter(subject=StudySubject.objects.filter(nd_number='cov-000111')[0])
 
@@ -127,7 +124,7 @@ class TestTnsCsvVisitReader(TestCase):
     def test_load_data_with_no_subject(self):
         filename = get_resource_path('tns_vouchers_import.csv')
         StudySubject.objects.filter(nd_number="cov-000111").delete()
-        visits = TnsCsvVisitImportReader(self.study.id).load_data(filename)
+        visits = TnsCsvVisitImportReader(self.visit_import_data).load_data(filename)
         self.assertEqual(3, len(visits))
         visit = Visit.objects.filter(id=visits[0].id)[0]
         self.assertEqual("cov-000111", visit.subject.nd_number)
@@ -137,7 +134,7 @@ class TestTnsCsvVisitReader(TestCase):
 
     def test_load_data_with_lab_id(self):
         filename = get_resource_path('tns_vouchers_lab_id_import.csv')
-        visits = TnsCsvVisitImportReader(self.study.id).load_data(filename)
+        visits = TnsCsvVisitImportReader(self.visit_import_data).load_data(filename)
         self.assertEqual(3, len(visits))
 
         visit = Visit.objects.filter(id=visits[0].id)[0]
@@ -156,10 +153,10 @@ class TestTnsCsvVisitReader(TestCase):
 
     def test_dont_add_links_for_existing_appointments(self):
         filename = get_resource_path('tns_vouchers_import.csv')
-        TnsCsvVisitImportReader(self.study.id).load_data(filename)
+        TnsCsvVisitImportReader(self.visit_import_data).load_data(filename)
         links = AppointmentTypeLink.objects.all().count()
 
-        TnsCsvVisitImportReader(self.study.id).load_data(filename)
+        TnsCsvVisitImportReader(self.visit_import_data).load_data(filename)
         self.assertEqual(links, AppointmentTypeLink.objects.all().count())
 
         self.assertEqual(0, self.get_warnings_count())