csv_delimiter is added to the database

smash/web/importer/csv_subject_import_reader.py

@@ -18,7 +18,7 @@ class CsvSubjectImportReader(SubjectImportReader):
     def load_data(self):
         study_subjects = []
         with open(self.import_data.get_absolute_file_path()) as csv_file:
-            reader = csv.reader(csv_file, delimiter=',')
+            reader = csv.reader(csv_file, delimiter=self.import_data.csv_delimiter)
             headers = next(reader, None)
             for row in reader:
                 subject = Subject()

smash/web/importer/csv_tns_subject_import_reader.py

@@ -19,7 +19,7 @@ class TnsCsvSubjectImportReader(SubjectImportReader):
     def load_data(self):
         study_subjects = []
         with open(self.import_data.get_absolute_file_path()) as csv_file:
-            reader = csv.reader((remove_bom(line) for line in csv_file), delimiter=';')
+            reader = csv.reader((remove_bom(line) for line in csv_file), delimiter=self.import_data.csv_delimiter)
             headers = next(reader, None)
             for row in reader:
                 subject = Subject()

smash/web/importer/csv_tns_visit_import_reader.py

@@ -43,7 +43,7 @@ class TnsCsvVisitImportReader:
 
         result = []
         with open(filename) as csv_file:
-            reader = csv.reader((remove_bom(line) for line in csv_file), delimiter=';')
+            reader = csv.reader((remove_bom(line) for line in csv_file), delimiter=self.visit_import_data.csv_delimiter)
             headers = next(reader, None)
             for row in reader:
                 # noinspection PyBroadException

smash/web/importer/importer.py

@@ -6,7 +6,7 @@ import traceback
 from django.contrib.auth.models import User
 
 from web.models import StudySubject, Subject, Provenance, Worker, ConfigurationItem
-from web.models.constants import GLOBAL_STUDY_ID, IMPORTER_USER
+from web.models.constants import IMPORTER_USER
 from .subject_import_reader import SubjectImportReader
 from .warning_counter import MsgCounterHandler
 
@@ -44,14 +44,10 @@ class Importer(object):
         self.study_subjects = self.reader.load_data()
         for study_subject in self.study_subjects:
             try:
-                if study_subject.study is None:
+                if study_subject.study != self.reader.import_data.study or study_subject.study.id is None:
                     self.problematic_count += 1
                     logger.warn("Empty study found. Ignoring")
                     continue
-                elif study_subject.study.id != GLOBAL_STUDY_ID:
-                    self.problematic_count += 1
-                    logger.warn("Empty study found. Ignoring: " + study_subject.study.id)
-                    continue
                 else:
                     self.import_study_subject(study_subject)
             except:

smash/web/importer/importer_cron_job.py

@@ -70,6 +70,7 @@ class SubjectImporterCronJob(CronJobBase):
                                          "<h3><font color='red'>There was a problem with importing data from file: " +
                                          filename + "</font></h3><pre>" + tb + "</pre>",
                                          email_recipients)
+                print(tb)
                 return "import crashed"
 
 

smash/web/migrations/0179_visitimportdata.py

@@ -17,6 +17,7 @@ class Migration(migrations.Migration):
                 ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                 ('filename', models.CharField(blank=True, default='', max_length=128, verbose_name='File used for automatic import')),
                 ('run_at_times', models.CharField(blank=True, default='', max_length=1024, verbose_name='At what time automatic import should run')),
+                ('csv_delimiter', models.CharField(blank=False, default=',', max_length=1, verbose_name='CSV delimiter')),
             ],
         ),
         migrations.CreateModel(

smash/web/migrations/0180_visitimportdata_migration.py

@@ -44,6 +44,7 @@ def create_visit_import_data(apps, schema_editor):
             entry.appointment_type = appointment_type[0]
     entry.filename = import_file
     entry.run_at_times = import_file_run_at
+    entry.delimiter = ';'
     entry.save()
 
 
@@ -70,6 +71,7 @@ def create_subject_import_data(apps, schema_editor):
 
     entry.filename = import_file
     entry.run_at_times = import_file_run_at
+    entry.delimiter = ';'
     entry.save()
 
 

smash/web/models/etl/etl.py

@@ -36,6 +36,13 @@ class EtlData(models.Model):
                                     blank=True
                                     )
 
+    csv_delimiter = models.CharField(max_length=1,
+                                     verbose_name='CSV delimiter',
+                                     default=',',
+                                     null=False,
+                                     blank=False
+                                     )
+
     def file_available(self) -> bool:
         if self.filename is None or self.filename == '':
             return False

smash/web/tests/importer/mock_reader.py

 import logging
+from typing import List
 
-from web.models import SubjectImportData
 from web.importer.subject_import_reader import SubjectImportReader
+from web.models import SubjectImportData, StudySubject
 
 logger = logging.getLogger(__name__)
 
 
 class MockReader(SubjectImportReader):
-    def __init__(self, study_subjects):
-        super().__init__(SubjectImportData())
+    def __init__(self, import_data: SubjectImportData, study_subjects: List[StudySubject]):
+        super().__init__(import_data)
         self.study_subjects = study_subjects
 
     def load_data(self):

smash/web/tests/importer/test_importer.py

@@ -6,9 +6,8 @@ from django.test import TestCase
 from django.utils import timezone
 
 from web.importer import Importer
-from web.models import Subject, StudySubject, Study, Provenance
-from web.models.constants import GLOBAL_STUDY_ID
-from web.tests.functions import create_study_subject
+from web.models import Subject, StudySubject, Provenance, SubjectImportData
+from web.tests.functions import create_study_subject, get_test_study
 from .mock_reader import MockReader
 
 logger = logging.getLogger(__name__)
@@ -17,7 +16,8 @@ logger = logging.getLogger(__name__)
 class TestImporter(TestCase):
 
     def setUp(self):
-        self.study = Study.objects.filter(id=GLOBAL_STUDY_ID)[0]
+        self.study = get_test_study()
+        self.visit_import_data = SubjectImportData.objects.create(study=self.study)
 
     def test_import_new_subject(self):
         study_subjects = []
@@ -30,7 +30,7 @@ class TestImporter(TestCase):
         study_subject.study = self.study
         study_subjects.append(study_subject)
 
-        importer = Importer(reader=MockReader(study_subjects))
+        importer = Importer(reader=MockReader(self.visit_import_data, study_subjects))
         subject_counter = Subject.objects.count()
         study_subject_counter = StudySubject.objects.count()
         provenance_counter = Provenance.objects.count()
@@ -51,7 +51,7 @@ class TestImporter(TestCase):
         study_subject.study = self.study
         study_subjects.append(study_subject)
 
-        importer = Importer(reader=MockReader(study_subjects))
+        importer = Importer(reader=MockReader(self.visit_import_data, study_subjects))
         study_subject_counter = StudySubject.objects.count()
         importer.execute()
 
@@ -66,7 +66,7 @@ class TestImporter(TestCase):
         study_subject.screening_number = "Cov-123456"
         study_subjects.append(study_subject)
 
-        importer = Importer(reader=MockReader(study_subjects))
+        importer = Importer(reader=MockReader(self.visit_import_data, study_subjects))
         study_subject_counter = StudySubject.objects.count()
         importer.execute()
 
@@ -89,7 +89,7 @@ class TestImporter(TestCase):
         study_subject.study = self.study
         study_subjects.append(study_subject)
 
-        importer = Importer(reader=MockReader(study_subjects))
+        importer = Importer(reader=MockReader(self.visit_import_data, study_subjects))
         provenance_counter = Provenance.objects.count()
         subject_counter = Subject.objects.count()
         study_subject_counter = StudySubject.objects.count()

smash/web/tests/importer/test_importer_cron_job.py

@@ -25,9 +25,11 @@ class TestCronJobImporter(TestCase):
         self.study.save()
         self.visit_import_data = VisitImportData.objects.create(study=self.study,
                                                                 appointment_type=create_appointment_type(),
-                                                                import_worker=create_worker())
+                                                                import_worker=create_worker(),
+                                                                csv_delimiter=';')
         self.subject_import_data = SubjectImportData.objects.create(study=self.study,
-                                                                    import_worker=create_worker())
+                                                                    import_worker=create_worker(),
+                                                                    csv_delimiter=';')
         CronJobLog.objects.all().delete()
 
     def test_import_without_configuration(self):