subject importer uses the same functionality that visit importer does

smash/web/importer/etl_common.py

 import codecs
 import datetime
+import logging
 from typing import Type, Optional
 
 from django.db import models
 
-from importer.subject_import_reader import logger
-from models import Provenance
-from models.etl.etl import EtlData
+from web.models import Provenance
+from web.models.etl.etl import EtlData
+
+logger = logging.getLogger(__name__)
 
 
 class EtlCommon:
@@ -87,4 +89,4 @@ class EtlCommon:
         if type(line) == str:
             return line[3:] if line.encode('utf8').startswith(codecs.BOM_UTF8) else line
         else:
-            return line[3:] if line.startswith(codecs.BOM_UTF8) else line
\ No newline at end of file
+            return line[3:] if line.startswith(codecs.BOM_UTF8) else line

smash/web/importer/importer.py

@@ -3,7 +3,7 @@ import logging
 import sys
 import traceback
 
-from web.models import StudySubject, Subject, Provenance
+from web.models import StudySubject, Subject
 from .etl_common import EtlCommon
 from .subject_import_reader import SubjectImportReader
 from .warning_counter import MsgCounterHandler
@@ -56,82 +56,27 @@ class Importer(EtlCommon):
                 if field.get_internal_type() == "CharField" or \
                         field.get_internal_type() == "DateField" or \
                         field.get_internal_type() is "BooleanField":
-                    new_value = getattr(study_subject.subject, field.name)
-                    if new_value is not None and new_value != "":
-                        old_value = getattr(db_study_subject.subject, field.name)
-                        description = '{} changed from "{}" to "{}"'.format(field, old_value, new_value)
-                        p = Provenance(modified_table=Subject._meta.db_table,
-                                       modified_table_id=db_study_subject.subject.id,
-                                       modification_author=self.etl_data.import_worker,
-                                       previous_value=old_value,
-                                       new_value=new_value,
-                                       modification_description=description,
-                                       modified_field=field,
-                                       )
-                        setattr(db_study_subject.subject, field.name, new_value)
-                        p.save()
-
+                    old_value = getattr(study_subject.subject, field.name)
+                    new_value = self.get_new_value(old_value, field.name, study_subject.subject)
+                    self.create_provenance_and_change_data(db_study_subject.subject, field.name, new_value, Subject)
             db_study_subject.subject.save()
 
             for field in StudySubject._meta.get_fields():
                 if field.get_internal_type() == "CharField" or \
                         field.get_internal_type() == "DateField" or \
                         field.get_internal_type() is "BooleanField":
-                    new_value = getattr(study_subject, field.name)
-                    if new_value is not None and new_value != "":
-                        old_value = getattr(db_study_subject, field.name)
-                        description = '{} changed from "{}" to "{}"'.format(field, old_value, new_value)
-                        p = Provenance(modified_table=Subject._meta.db_table,
-                                       modified_table_id=db_study_subject.id,
-                                       modification_author=self.etl_data.import_worker,
-                                       previous_value=old_value,
-                                       new_value=new_value,
-                                       modification_description=description,
-                                       modified_field=field,
-                                       )
-                        setattr(db_study_subject, field.name, new_value)
-                        p.save()
+                    old_value = getattr(study_subject, field.name)
+                    new_value = self.get_new_value(old_value, field.name, study_subject.subject)
+                    self.create_provenance_and_change_data(db_study_subject, field.name, new_value, StudySubject)
             db_study_subject.save()
+
             self.merged_count += 1
         else:
             study_subject.subject.save()
-            study_subject.subject = Subject.objects.filter(id=study_subject.subject.id)[0]
+            study_subject.subject = Subject.objects.get(pk=study_subject.subject.id)
             study_subject.save()
-
-            for field in Subject._meta.get_fields():
-                if field.get_internal_type() == "CharField" or \
-                        field.get_internal_type() == "DateField" or \
-                        field.get_internal_type() is "BooleanField":
-                    new_value = getattr(study_subject.subject, field.name)
-                    if new_value is not None and new_value != "":
-                        description = '{} changed from "{}" to "{}"'.format(field, '', new_value)
-                        p = Provenance(modified_table=Subject._meta.db_table,
-                                       modified_table_id=study_subject.subject.id,
-                                       modification_author=self.etl_data.import_worker,
-                                       previous_value='',
-                                       new_value=new_value,
-                                       modification_description=description,
-                                       modified_field=field,
-                                       )
-                        p.save()
-
-            for field in StudySubject._meta.get_fields():
-                if field.get_internal_type() == "CharField" or \
-                        field.get_internal_type() == "DateField" or \
-                        field.get_internal_type() is "BooleanField":
-                    new_value = getattr(study_subject, field.name)
-                    if new_value is not None and new_value != "":
-                        description = '{} changed from "{}" to "{}"'.format(field, '', new_value)
-                        p = Provenance(modified_table=Subject._meta.db_table,
-                                       modified_table_id=study_subject.id,
-                                       modification_author=self.etl_data.import_worker,
-                                       previous_value='',
-                                       new_value=new_value,
-                                       modification_description=description,
-                                       modified_field=field,
-                                       )
-                        p.save()
-
+            self.create_provenance_for_new_object(Subject, study_subject.subject)
+            self.create_provenance_for_new_object(StudySubject, study_subject)
             self.added_count += 1
 
     def get_summary(self):

smash/web/importer/subject_import_reader.py

 import logging
 from typing import List
 
-from importer.etl_common import EtlCommon
+from web.importer.etl_common import EtlCommon
 from web.models import SubjectImportData
 from web.models.study_subject import StudySubject