Skip to content
Snippets Groups Projects
Commit 9bce8179 authored by Piotr Gawron's avatar Piotr Gawron
Browse files

TnsCsvVisitImportReader renamed to CsvVisitImportReader

parent ccb8ab42
No related branches found
No related tags found
1 merge request!275Resolve "update automatic visit/subject importer"
Hide whitespace changes
Inline Side-by-side
smash/web/importer/__init__.py
+ 2
− 2
View file @ 9bce8179
from .csv_subject_import_reader import CsvSubjectImportReader
from .csv_tns_visit_import_reader import TnsCsvVisitImportReader
from .csv_visit_import_reader import CsvVisitImportReader
from .exporter import SubjectExporter, VisitExporter
from .exporter_cron_job import SubjectExporterCronJob, VisitExporterCronJob
from .importer import Importer
......@@ -9,4 +9,4 @@ from .warning_counter import MsgCounterHandler
__all__ = [Importer, SubjectImportReader, CsvSubjectImportReader, SubjectImporterCronJob, VisitImporterCronJob,
SubjectExporter, VisitExporter, SubjectExporterCronJob, VisitExporterCronJob,
TnsCsvVisitImportReader, MsgCounterHandler]
CsvVisitImportReader, MsgCounterHandler]
smash/web/importer/csv_tns_visit_import_reader.py → smash/web/importer/csv_visit_import_reader.py
+ 1
− 1
View file @ 9bce8179
......@@ -19,7 +19,7 @@ class EtlException(Exception):
pass
class TnsCsvVisitImportReader(EtlCommon):
class CsvVisitImportReader(EtlCommon):
def __init__(self, data: VisitImportData):
super().__init__(data)
self.import_data = data
......
smash/web/importer/importer_cron_job.py
+ 2
− 2
View file @ 9bce8179
......@@ -13,7 +13,7 @@ from web.models import ConfigurationItem, Study, VisitImportData, SubjectImportD
from web.models.constants import CRON_JOB_TIMEOUT, DEFAULT_FROM_EMAIL, GLOBAL_STUDY_ID
from web.smash_email import EmailSender
from . import CsvSubjectImportReader
from .csv_tns_visit_import_reader import TnsCsvVisitImportReader
from .csv_visit_import_reader import CsvVisitImportReader
from .importer import Importer
logger = logging.getLogger(__name__)
......@@ -109,7 +109,7 @@ class VisitImporterCronJob(CronJobBase):
# noinspection PyBroadException
try:
importer = TnsCsvVisitImportReader(import_data)
importer = CsvVisitImportReader(import_data)
importer.load_data()
email_body = importer.get_summary()
EmailSender().send_email(email_title,
......
smash/web/tests/importer/test_tns_csv_visit_import_reader.py
+ 24
− 24
View file @ 9bce8179
......@@ -7,8 +7,8 @@ from django.conf import settings
from django.test import TestCase
from django.utils import timezone
from web.importer import TnsCsvVisitImportReader, MsgCounterHandler
from web.importer.csv_tns_visit_import_reader import EtlException
from web.importer import CsvVisitImportReader, MsgCounterHandler
from web.importer.csv_visit_import_reader import EtlException
from web.models import Appointment, Visit, StudySubject, AppointmentTypeLink, AppointmentType, VisitImportData, \
Provenance
from web.tests.functions import get_resource_path, create_study_subject, create_appointment_type, create_location, \
......@@ -45,7 +45,7 @@ class TestTnsCsvVisitReader(TestCase):
def test_load_data(self):
self.visit_import_data.filename = 'tns_vouchers_import.csv'
visits = TnsCsvVisitImportReader(self.visit_import_data).load_data()
visits = CsvVisitImportReader(self.visit_import_data).load_data()
self.assertEqual(3, len(visits))
visit = Visit.objects.filter(id=visits[0].id)[0]
self.assertEqual("cov-000111", visit.subject.nd_number)
......@@ -69,7 +69,7 @@ class TestTnsCsvVisitReader(TestCase):
datetime_begin=timezone.now(),
datetime_end=timezone.now())
visit = TnsCsvVisitImportReader(self.visit_import_data).load_data()[0]
visit = CsvVisitImportReader(self.visit_import_data).load_data()[0]
self.assertEqual(1, Provenance.objects.filter(modified_table=Visit._meta.db_table,
modified_table_id=visit.id,
......@@ -94,7 +94,7 @@ class TestTnsCsvVisitReader(TestCase):
AppointmentTypeLink.objects.create(appointment=old_appointment,
appointment_type=self.visit_import_data.appointment_type)
TnsCsvVisitImportReader(self.visit_import_data).load_data()
CsvVisitImportReader(self.visit_import_data).load_data()
self.assertEqual(1, Provenance.objects.filter(modified_table=Appointment._meta.db_table,
modified_table_id=old_appointment.id,
......@@ -110,7 +110,7 @@ class TestTnsCsvVisitReader(TestCase):
def test_data_provenance_for_create_appointment_load_data(self):
self.visit_import_data.filename = 'tns_vouchers_import.csv'
visit = TnsCsvVisitImportReader(self.visit_import_data).load_data()[0]
visit = CsvVisitImportReader(self.visit_import_data).load_data()[0]
appointment = visit.appointment_set.all()[0]
self.assertEqual(0, Provenance.objects.filter(modified_table=Appointment._meta.db_table,
......@@ -127,7 +127,7 @@ class TestTnsCsvVisitReader(TestCase):
def test_data_provenance_for_create_visit_load_data(self):
self.visit_import_data.filename = 'tns_vouchers_import.csv'
visit = TnsCsvVisitImportReader(self.visit_import_data).load_data()[0]
visit = CsvVisitImportReader(self.visit_import_data).load_data()[0]
self.assertEqual(0, Provenance.objects.filter(modified_table=Visit._meta.db_table,
modified_table_id=visit.id,
......@@ -146,7 +146,7 @@ class TestTnsCsvVisitReader(TestCase):
datetime_end=timezone.now(),
datetime_begin=timezone.now(),
visit_number=1)
visits = TnsCsvVisitImportReader(self.visit_import_data).load_data()
visits = CsvVisitImportReader(self.visit_import_data).load_data()
visit = Visit.objects.filter(id=visits[0].id)[0]
self.assertEqual("cov-000111", visit.subject.nd_number)
......@@ -172,7 +172,7 @@ class TestTnsCsvVisitReader(TestCase):
AppointmentTypeLink.objects.create(appointment_id=appointment.id,
appointment_type=AppointmentType.objects.filter(code="SAMPLE_2")[0])
visits = TnsCsvVisitImportReader(self.visit_import_data).load_data()
visits = CsvVisitImportReader(self.visit_import_data).load_data()
visit = Visit.objects.filter(id=visits[0].id)[0]
self.assertEqual("cov-000111", visit.subject.nd_number)
......@@ -191,7 +191,7 @@ class TestTnsCsvVisitReader(TestCase):
def test_load_data_with_visit_and_no_previous_visits(self):
self.visit_import_data.filename = 'tns_vouchers_3_import.csv'
visits = TnsCsvVisitImportReader(self.visit_import_data).load_data()
visits = CsvVisitImportReader(self.visit_import_data).load_data()
subject_visits = Visit.objects.filter(subject=StudySubject.objects.filter(nd_number='cov-000111')[0])
......@@ -213,7 +213,7 @@ class TestTnsCsvVisitReader(TestCase):
def test_load_data_with_no_subject(self):
self.visit_import_data.filename = 'tns_vouchers_import.csv'
StudySubject.objects.filter(nd_number="cov-000111").delete()
visits = TnsCsvVisitImportReader(self.visit_import_data).load_data()
visits = CsvVisitImportReader(self.visit_import_data).load_data()
self.assertEqual(3, len(visits))
visit = Visit.objects.filter(id=visits[0].id)[0]
self.assertEqual("cov-000111", visit.subject.nd_number)
......@@ -223,65 +223,65 @@ class TestTnsCsvVisitReader(TestCase):
def test_dont_add_links_for_existing_appointments(self):
self.visit_import_data.filename = 'tns_vouchers_import.csv'
TnsCsvVisitImportReader(self.visit_import_data).load_data()
CsvVisitImportReader(self.visit_import_data).load_data()
links = AppointmentTypeLink.objects.all().count()
TnsCsvVisitImportReader(self.visit_import_data).load_data()
CsvVisitImportReader(self.visit_import_data).load_data()
self.assertEqual(links, AppointmentTypeLink.objects.all().count())
self.assertEqual(0, self.get_warnings_count())
def test_get_study_subject_by_invalid_id_column(self):
self.assertRaises(EtlException,
TnsCsvVisitImportReader(self.visit_import_data).get_study_subject_id, {'invalid_id': 'x001'})
CsvVisitImportReader(self.visit_import_data).get_study_subject_id, {'invalid_id': 'x001'})
def test_get_study_subject_by_valid_id_column(self):
subject_id = TnsCsvVisitImportReader(self.visit_import_data).get_study_subject_id({'donor_id': 'x001'})
subject_id = CsvVisitImportReader(self.visit_import_data).get_study_subject_id({'donor_id': 'x001'})
self.assertEqual('x001', subject_id)
def test_get_visit_date_by_invalid_id_column(self):
self.assertRaises(EtlException,
TnsCsvVisitImportReader(self.visit_import_data).get_visit_date,
CsvVisitImportReader(self.visit_import_data).get_visit_date,
{'invalid_date': '19901010'})
def test_get_visit_date_by_valid_id_column(self):
visit_date = TnsCsvVisitImportReader(self.visit_import_data).get_visit_date({'dateofvisit': '19901010'})
visit_date = CsvVisitImportReader(self.visit_import_data).get_visit_date({'dateofvisit': '19901010'})
self.assertEqual(1990, visit_date.year)
def test_get_location_by_invalid_id_column(self):
self.assertRaises(EtlException,
TnsCsvVisitImportReader(self.visit_import_data).extract_location, {'invalid_id': 'x001'})
CsvVisitImportReader(self.visit_import_data).extract_location, {'invalid_id': 'x001'})
def test_get_location_by_valid_id_column(self):
location = TnsCsvVisitImportReader(self.visit_import_data).extract_location({'adressofvisit': 'x001'})
location = CsvVisitImportReader(self.visit_import_data).extract_location({'adressofvisit': 'x001'})
self.assertEqual('x001', location.name)
self.assertEqual(1, self.get_warnings_count())
def test_get_visit_number_by_invalid_id_column(self):
self.assertRaises(EtlException,
TnsCsvVisitImportReader(self.visit_import_data).get_visit_number, {'invalid_id': '1'})
CsvVisitImportReader(self.visit_import_data).get_visit_number, {'invalid_id': '1'})
def test_get_visit_number_by_valid_id_column(self):
self.visit_import_data.study.redcap_first_visit_number = 0
self.visit_import_data.study.save()
visit_number = TnsCsvVisitImportReader(self.visit_import_data).get_visit_number({'visit_id': '1'})
visit_number = CsvVisitImportReader(self.visit_import_data).get_visit_number({'visit_id': '1'})
# normalized visit number is from 1
self.assertEqual(2, visit_number)
def test_get_visit_number_by_valid_id_column_started_from_one(self):
self.visit_import_data.study.redcap_first_visit_number = 1
self.visit_import_data.study.save()
visit_number = TnsCsvVisitImportReader(self.visit_import_data).get_visit_number({'visit_id': '1'})
visit_number = CsvVisitImportReader(self.visit_import_data).get_visit_number({'visit_id': '1'})
self.assertEqual(1, visit_number)
def test_get_study_subject_by_id_for_existing_subject(self):
reader = TnsCsvVisitImportReader(self.visit_import_data)
reader = CsvVisitImportReader(self.visit_import_data)
study_subject = StudySubject.objects.get(nd_number='cov-000111')
study_subject_from_data = reader.get_study_subject_by_id('cov-000111')
self.assertEqual(study_subject, study_subject_from_data)
def test_get_study_subject_by_id_for_existing_subject_in_different_study(self):
reader = TnsCsvVisitImportReader(self.visit_import_data)
reader = CsvVisitImportReader(self.visit_import_data)
study_subject = create_study_subject(nd_number='cov-09458', study=create_study())
study_subject_from_data = reader.get_study_subject_by_id('cov-09458')
......
smash/web/views/etl.py
+ 2
− 2
View file @ 9bce8179
......@@ -7,7 +7,7 @@ from django.shortcuts import redirect, get_object_or_404
from web.decorators import PermissionDecorator
from web.forms.subject_import_data_form import SubjectImportDataEditForm
from web.forms.visit_import_data_form import VisitImportDataEditForm
from web.importer import Importer, CsvSubjectImportReader, TnsCsvVisitImportReader
from web.importer import Importer, CsvSubjectImportReader, CsvVisitImportReader
from web.importer.log_storage import LogStorageHandler
from web.models import Study, VisitImportData, SubjectImportData
from web.views import wrap_response
......@@ -38,7 +38,7 @@ def import_visit_execute(request, study_id, import_id):
study = get_object_or_404(Study, id=study_id)
import_data = get_object_or_404(VisitImportData, id=import_id)
if import_data.file_available():
reader = TnsCsvVisitImportReader(import_data)
reader = CsvVisitImportReader(import_data)
log_storage = LogStorageHandler()
logging.getLogger('').addHandler(log_storage)
reader.load_data()
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment