importer.py

# coding=utf-8
import logging
import sys
import traceback

from web.models import StudySubject, Subject
from .etl_common import EtlCommon
from .subject_import_reader import SubjectImportReader
from .warning_counter import MsgCounterHandler

logger = logging.getLogger(__name__)


class Importer(EtlCommon):
    def __init__(self, reader: SubjectImportReader):
        super().__init__(reader.etl_data)
        self.reader = reader
        self.added_count = 0
        self.problematic_count = 0
        self.merged_count = 0
        self.warning_count = 0
        self.study_subjects = []

    def execute(self):
        self.added_count = 0
        self.problematic_count = 0
        self.merged_count = 0
        self.warning_count = 0

        warning_counter = MsgCounterHandler()
        logging.getLogger('').addHandler(warning_counter)

        self.study_subjects = self.reader.load_data()
        for study_subject in self.study_subjects:
            # noinspection PyBroadException
            try:
                if study_subject.study != self.reader.import_data.study or study_subject.study.id is None:
                    self.problematic_count += 1
                    logger.warning("Empty study found. Ignoring")
                    continue
                else:
                    self.import_study_subject(study_subject)
            except BaseException as e:
                self.problematic_count += 1
                traceback.print_exc(file=sys.stdout)
                logger.error("Problem with importing study subject: " + str(study_subject.screening_number) + "," +
                             str(study_subject.nd_number), exc_info=e)
        if "WARNING" in warning_counter.level2count:
            self.warning_count = warning_counter.level2count["WARNING"]
        logging.getLogger('').removeHandler(warning_counter)

    def import_study_subject(self, study_subject: StudySubject):
        db_study_subject = StudySubject.objects.filter(nd_number=study_subject.nd_number).first()
        if db_study_subject is not None:
            for field in Subject._meta.get_fields():
                if field.get_internal_type() == "CharField" or \
                        field.get_internal_type() == "DateField" or \
                        field.get_internal_type() is "BooleanField":
                    old_value = getattr(db_study_subject.subject, field.name)
                    new_value = self.get_new_value(old_value, getattr(study_subject.subject, field.name))
                    self.create_provenance_and_change_data(db_study_subject.subject, field.name, new_value, Subject)
            db_study_subject.subject.save()

            for field in StudySubject._meta.get_fields():
                if field.get_internal_type() == "CharField" or \
                        field.get_internal_type() == "DateField" or \
                        field.get_internal_type() is "BooleanField":
                    old_value = getattr(db_study_subject, field.name)
                    new_value = self.get_new_value(old_value, getattr(study_subject, field.name))
                    self.create_provenance_and_change_data(db_study_subject, field.name, new_value, StudySubject)
            db_study_subject.save()

            self.merged_count += 1
        else:
            study_subject.subject.save()
            study_subject.subject = Subject.objects.get(pk=study_subject.subject.id)
            study_subject.save()
            self.create_provenance_for_new_object(Subject, study_subject.subject)
            self.create_provenance_for_new_object(StudySubject, study_subject)
            self.added_count += 1

    def get_summary(self):
        result = "<p>Number of entries: <b>" + str(len(self.study_subjects)) + "</b></p>" + \
                 "<p>Number of successfully added entries: <b>" + str(self.added_count) + "</b></p>" + \
                 "<p>Number of successfully merged entries: <b>" + str(self.merged_count) + "</b></p>"
        style = ''
        if self.problematic_count > 0:
            style = ' color="red" '
        result += "<p><font " + style + ">Number of problematic entries: <b>" + str(
            self.problematic_count) + "</b></font></p>"
        style = ''
        if self.warning_count > 0:
            style = ' color="brown" '
        result += "<p><font " + style + ">Number of raised warnings: <b>" + str(self.warning_count) + "</b></font></p>"

        return result