# coding=utf-8 import logging import sys import traceback from web.models import StudySubject, Subject from .etl_common import EtlCommon from .subject_import_reader import SubjectImportReader from .warning_counter import MsgCounterHandler logger = logging.getLogger(__name__) class Importer(EtlCommon): def __init__(self, reader: SubjectImportReader): super().__init__(reader.etl_data) self.reader = reader self.added_count = 0 self.problematic_count = 0 self.merged_count = 0 self.warning_count = 0 self.study_subjects = [] def execute(self): self.added_count = 0 self.problematic_count = 0 self.merged_count = 0 self.warning_count = 0 warning_counter = MsgCounterHandler() logging.getLogger('').addHandler(warning_counter) self.study_subjects = self.reader.load_data() for study_subject in self.study_subjects: # noinspection PyBroadException try: if study_subject.study != self.reader.import_data.study or study_subject.study.id is None: self.problematic_count += 1 logger.warning("Empty study found. Ignoring") continue else: self.import_study_subject(study_subject) except BaseException as e: self.problematic_count += 1 traceback.print_exc(file=sys.stdout) logger.error("Problem with importing study subject: " + str(study_subject.screening_number) + "," + str(study_subject.nd_number), exc_info=e) if "WARNING" in warning_counter.level2count: self.warning_count = warning_counter.level2count["WARNING"] logging.getLogger('').removeHandler(warning_counter) def import_study_subject(self, study_subject: StudySubject): db_study_subject = StudySubject.objects.filter(nd_number=study_subject.nd_number).first() if db_study_subject is not None: for field in Subject._meta.get_fields(): if field.get_internal_type() == "CharField" or \ field.get_internal_type() == "DateField" or \ field.get_internal_type() is "BooleanField": old_value = getattr(db_study_subject.subject, field.name) new_value = self.get_new_value(old_value, getattr(study_subject.subject, field.name)) self.create_provenance_and_change_data(db_study_subject.subject, field.name, new_value, Subject) db_study_subject.subject.save() for field in StudySubject._meta.get_fields(): if field.get_internal_type() == "CharField" or \ field.get_internal_type() == "DateField" or \ field.get_internal_type() is "BooleanField": old_value = getattr(db_study_subject, field.name) new_value = self.get_new_value(old_value, getattr(study_subject, field.name)) self.create_provenance_and_change_data(db_study_subject, field.name, new_value, StudySubject) db_study_subject.save() self.merged_count += 1 else: study_subject.subject.save() study_subject.subject = Subject.objects.get(pk=study_subject.subject.id) study_subject.save() self.create_provenance_for_new_object(Subject, study_subject.subject) self.create_provenance_for_new_object(StudySubject, study_subject) self.added_count += 1 def get_summary(self): result = "<p>Number of entries: <b>" + str(len(self.study_subjects)) + "</b></p>" + \ "<p>Number of successfully added entries: <b>" + str(self.added_count) + "</b></p>" + \ "<p>Number of successfully merged entries: <b>" + str(self.merged_count) + "</b></p>" style = '' if self.problematic_count > 0: style = ' color="red" ' result += "<p><font " + style + ">Number of problematic entries: <b>" + str( self.problematic_count) + "</b></font></p>" style = '' if self.warning_count > 0: style = ' color="brown" ' result += "<p><font " + style + ">Number of raised warnings: <b>" + str(self.warning_count) + "</b></font></p>" return result