Merge branch 'redcap-sync' into 'master'

Redcap sync

See merge request NCER-PD/scheduling-system!244

smash/smash/settings.py

@@ -76,7 +76,8 @@ CRON_CLASSES = [
     'web.views.kit.KitRequestEmailSendJob',
     'web.redcap_connector.RedCapRefreshJob',
     'web.views.voucher.ExpireVouchersJob',
-    'web.importer.exporter_cron_job.ExporterCronJob',
+    'web.importer.exporter_cron_job.SubjectExporterCronJob',
+    'web.importer.exporter_cron_job.VisitExporterCronJob',
     'web.importer.importer_cron_job.SubjectImporterCronJob',
     'web.importer.importer_cron_job.VisitImporterCronJob'
 ]

smash/web/importer/__init__.py

 from csv_subject_import_reader import CsvSubjectImportReader
 from csv_tns_subject_import_reader import TnsCsvSubjectImportReader
 from csv_tns_visit_import_reader import TnsCsvVisitImportReader
-from exporter import Exporter
-from exporter_cron_job import ExporterCronJob
+from exporter import SubjectExporter, VisitExporter
+from exporter_cron_job import SubjectExporterCronJob, VisitExporterCronJob
 from importer import Importer
 from importer_cron_job import SubjectImporterCronJob, VisitImporterCronJob
 from subject_import_reader import SubjectImportReader
 from warning_counter import MsgCounterHandler
 
 __all__ = [Importer, SubjectImportReader, CsvSubjectImportReader, SubjectImporterCronJob, VisitImporterCronJob,
-           Exporter, ExporterCronJob, TnsCsvSubjectImportReader, TnsCsvVisitImportReader, MsgCounterHandler]
+           SubjectExporter, VisitExporter, SubjectExporterCronJob, VisitExporterCronJob, TnsCsvSubjectImportReader,
+           TnsCsvVisitImportReader, MsgCounterHandler]

smash/web/importer/csv_tns_visit_import_reader.py

 # coding=utf-8
+import codecs
 import csv
 import datetime
 import logging
 import sys
 import traceback
-import codecs
 
 import pytz
 from django.conf import settings
@@ -100,9 +100,9 @@ class TnsCsvVisitImportReader:
                     else:
                         appointment = Appointment.objects.create(visit=visit, length=60, datetime_when=date,
                                                                  location=location)
-                    if self.appointment_type is not None:
-                        AppointmentTypeLink.objects.create(appointment_id=appointment.id,
-                                                           appointment_type=self.appointment_type)
+                        if self.appointment_type is not None:
+                            AppointmentTypeLink.objects.create(appointment_id=appointment.id,
+                                                               appointment_type=self.appointment_type)
                     self.processed_count += 1
                 except:
                     self.problematic_count += 1

smash/web/importer/exporter.py

 # coding=utf-8
 import csv
 import logging
+import sys
+import traceback
 
 from warning_counter import MsgCounterHandler
-from web.models import StudySubject
+from web.models import StudySubject, Appointment
 
 logger = logging.getLogger(__name__)
 
 
-class Exporter(object):
+class SubjectExporter(object):
     def __init__(self, filename):
         # type: (str) -> None
         self.filename = filename
@@ -48,3 +50,50 @@ class Exporter(object):
         for study_subject in study_subjects:
             result.append([study_subject.nd_number])
         return result
+
+
+class VisitExporter(object):
+    def __init__(self, filename):
+        # type: (str) -> None
+        self.filename = filename
+        self.exported_count = 0
+        self.warning_count = 0
+
+    def execute(self):
+        self.exported_count = 0
+        self.warning_count = 0
+
+        warning_counter = MsgCounterHandler()
+        logging.getLogger('').addHandler(warning_counter)
+
+        with open(self.filename, 'w') as csv_file:
+            data = self.get_appointments_as_array()
+            writer = csv.writer(csv_file, quotechar=str(u'"'))
+            for row in data:
+                writer.writerow([s.encode("utf-8") for s in row])
+                self.exported_count += 1
+
+        if "WARNING" in warning_counter.level2count:
+            self.warning_count = warning_counter.level2count["WARNING"]
+        logging.getLogger('').removeHandler(warning_counter)
+
+    def get_summary(self):
+        result = "<p>Number of entries: <b>" + str(self.exported_count) + "</b></p>"
+        style = ''
+        if self.warning_count > 0:
+            style = ' color="brown" '
+        result += "<p><font " + style + ">Number of raised warnings: <b>" + str(self.warning_count) + "</b></font></p>"
+
+        return result
+
+    def get_appointments_as_array(self):
+        result = []
+        appointments = Appointment.objects.filter(status=Appointment.APPOINTMENT_STATUS_FINISHED)
+        for appointment in appointments:
+            try:
+                if appointment.visit is not None:
+                    result.append([appointment.visit.subject.nd_number, str(appointment.visit.visit_number - 1)])
+            except:
+                traceback.print_exc(file=sys.stdout)
+                logger.warn("Problem with exporting appointment: " + str(appointment.id))
+        return result

smash/web/importer/exporter_cron_job.py

@@ -6,21 +6,21 @@ import timeout_decorator
 from django.conf import settings
 from django_cron import CronJobBase, Schedule
 
-from exporter import Exporter
+from exporter import SubjectExporter, VisitExporter
 from web.models.constants import CRON_JOB_TIMEOUT
 from ..smash_email import EmailSender
 
 logger = logging.getLogger(__name__)
 
 
-class ExporterCronJob(CronJobBase):
+class SubjectExporterCronJob(CronJobBase):
     RUN_AT_TIMES = getattr(settings, "EXPORT_RUN_AT", ['23:55'])
     schedule = Schedule(run_at_times=RUN_AT_TIMES)
-    code = 'web.import_daily_job'  # a unique code
+    code = 'web.export_subject_daily_job'  # a unique code
 
     @timeout_decorator.timeout(CRON_JOB_TIMEOUT)
     def do(self):
-        email_title = "Daily export"
+        email_title = "Daily subject export"
         email_recipients = getattr(settings, "DEFAULT_FROM_EMAIL", None)
 
         filename = getattr(settings, "DAILY_SUBJECT_EXPORT_FILE", None)
@@ -28,9 +28,42 @@ class ExporterCronJob(CronJobBase):
         if filename is None:
             logger.info("Exporting subjects skipped. File not defined ")
             return "export file not defined"
-        logger.info("Exporting subjects from file: " + filename)
+        logger.info("Exporting subjects to file: " + filename)
         try:
-            exporter = Exporter(settings.DAILY_SUBJECT_EXPORT_FILE)
+            exporter = SubjectExporter(filename)
+            exporter.execute()
+            email_body = exporter.get_summary()
+            EmailSender().send_email(email_title,
+                                     "<h3>Data was successfully exported to file: " + filename + "</h3>" + email_body,
+                                     email_recipients)
+            return "export is successful"
+
+        except:
+            tb = traceback.format_exc()
+            EmailSender().send_email(email_title,
+                                     "<h3><font color='red'>There was a problem with exporting data to file: " + filename + "</font></h3><pre>" + tb + "</pre>",
+                                     email_recipients)
+            return "export crashed"
+
+
+class VisitExporterCronJob(CronJobBase):
+    RUN_AT_TIMES = getattr(settings, "EXPORT_RUN_AT", ['23:55'])
+    schedule = Schedule(run_at_times=RUN_AT_TIMES)
+    code = 'web.export_visit_daily_job'  # a unique code
+
+    @timeout_decorator.timeout(CRON_JOB_TIMEOUT)
+    def do(self):
+        email_title = "Daily visit export"
+        email_recipients = getattr(settings, "DEFAULT_FROM_EMAIL", None)
+
+        filename = getattr(settings, "DAILY_VISIT_EXPORT_FILE", None)
+
+        if filename is None:
+            logger.info("Exporting visit skipped. File not defined ")
+            return "export file not defined"
+        logger.info("Exporting visits to file: " + filename)
+        try:
+            exporter = VisitExporter(filename)
             exporter.execute()
             email_body = exporter.get_summary()
             EmailSender().send_email(email_title,

smash/web/tests/importer/test_exporter.py

@@ -6,7 +6,7 @@ import logging
 from django.test import TestCase
 
 from web.tests.functions import create_study_subject
-from web.importer import Exporter
+from web.importer import SubjectExporter
 from web.models import Subject, StudySubject, Study, Provenance
 from web.models.constants import GLOBAL_STUDY_ID
 
@@ -21,7 +21,7 @@ class TestExporter(TestCase):
     def test_export_not_excluded(self):
         create_study_subject()
 
-        exporter = Exporter(filename="empty.csv")
+        exporter = SubjectExporter(filename="empty.csv")
         exporter.execute()
 
         self.assertEqual(0, exporter.exported_count)
@@ -32,7 +32,7 @@ class TestExporter(TestCase):
         subject.excluded=True
         subject.save()
 
-        exporter = Exporter(filename="empty.csv")
+        exporter = SubjectExporter(filename="empty.csv")
         exporter.execute()
 
         self.assertEqual(1, exporter.exported_count)

smash/web/tests/importer/test_exporter_cron_job.py

@@ -6,7 +6,7 @@ import tempfile
 from django.conf import settings
 from django.test import TestCase
 
-from web.importer import ExporterCronJob
+from web.importer import SubjectExporterCronJob
 
 logger = logging.getLogger(__name__)
 from django.core import mail
@@ -24,7 +24,7 @@ class TestCronJobExporter(TestCase):
     def test_import_without_configuration(self):
         CronJobLog.objects.all().delete()
 
-        job = ExporterCronJob()
+        job = SubjectExporterCronJob()
 
         status = job.do()
 
@@ -37,7 +37,7 @@ class TestCronJobExporter(TestCase):
         setattr(settings, "DAILY_SUBJECT_EXPORT_FILE", tmp)
         CronJobLog.objects.all().delete()
 
-        job = ExporterCronJob()
+        job = SubjectExporterCronJob()
 
         status = job.do()
 

smash/web/tests/importer/test_tns_csv_visit_import_reader.py

@@ -151,6 +151,16 @@ class TestTnsCsvVisitReader(TestCase):
 
         self.assertEquals(3, self.get_warnings_count())
 
+    def test_dont_add_links_for_existing_appointments(self):
+        filename = get_resource_path('tns_vouchers_import.csv')
+        TnsCsvVisitImportReader().load_data(filename)
+        links = AppointmentTypeLink.objects.all().count()
+
+        TnsCsvVisitImportReader().load_data(filename)
+        self.assertEquals(links, AppointmentTypeLink.objects.all().count())
+
+        self.assertEquals(0, self.get_warnings_count())
+
     def get_warnings_count(self):
         if "WARNING" in self.warning_counter.level2count:
             return self.warning_counter.level2count["WARNING"]