diff --git a/CHANGELOG b/CHANGELOG index 36cc1d71a86e37e4015a1e785329a378fba31f63..a74c0556be72e45c7affd4878a736943196ac18f 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,7 +1,8 @@ smasch (1.4.3-1) stable; urgency=medium + * make export faster * move copyright note to settings - -- Carlos Vega <carlos.vega@lih.lu> Fri, 10 May 2024 15:47:00 +0200 + -- Carlos Vega <carlos.vega@lih.lu> Tue, 14 May 2024 10:07:00 +0200 smasch (1.4.2-1) stable; urgency=medium * dependencies: updated package.json and package-lock.json diff --git a/smash/web/models/study_subject.py b/smash/web/models/study_subject.py index 78d65930195e3ceac62ddaac4abb73735d8f8143..3a3ca3ef6e4a3df1425f4e87286948a96a7c1142 100644 --- a/smash/web/models/study_subject.py +++ b/smash/web/models/study_subject.py @@ -4,12 +4,13 @@ import re from typing import Optional from django.db import models +from django.db.models import FilteredRelation, Q from django.db.models.signals import post_save from django.dispatch import receiver - -from web.models import VoucherType, Appointment, Location, Visit, Provenance +from web.models import Appointment, Location, Provenance, Visit, VoucherType from web.models.constants import BOOL_CHOICES, FILE_STORAGE -from web.models.custom_data import CustomStudySubjectValue, CustomStudySubjectField +from web.models.custom_data import (CustomStudySubjectField, + CustomStudySubjectValue) logger = logging.getLogger(__name__) @@ -31,7 +32,9 @@ class StudySubject(models.Model): visit.save() def finish_all_appointments(self): - appointments = Appointment.objects.filter(visit__subject=self, status=Appointment.APPOINTMENT_STATUS_SCHEDULED) + appointments = Appointment.objects.filter( + visit__subject=self, status=Appointment.APPOINTMENT_STATUS_SCHEDULED + ) for appointment in appointments: appointment.status = Appointment.APPOINTMENT_STATUS_CANCELLED appointment.save() @@ -52,19 +55,37 @@ class StudySubject(models.Model): self.finish_all_appointments() subject = models.ForeignKey( - "web.Subject", verbose_name="Subject", editable=False, null=False, on_delete=models.CASCADE + "web.Subject", + verbose_name="Subject", + editable=False, + null=False, + on_delete=models.CASCADE, ) - study = models.ForeignKey("web.Study", verbose_name="Study", editable=False, null=False, on_delete=models.CASCADE) + study = models.ForeignKey( + "web.Study", + verbose_name="Study", + editable=False, + null=False, + on_delete=models.CASCADE, + ) - postponed = models.BooleanField(choices=BOOL_CHOICES, verbose_name="Postponed", default=False) + postponed = models.BooleanField( + choices=BOOL_CHOICES, verbose_name="Postponed", default=False + ) datetime_contact_reminder = models.DateTimeField( null=True, blank=True, verbose_name="Please make a contact on", ) - type = models.ForeignKey("web.SubjectType", null=False, blank=False, on_delete=models.PROTECT, verbose_name="Type") + type = models.ForeignKey( + "web.SubjectType", + null=False, + blank=False, + on_delete=models.PROTECT, + verbose_name="Type", + ) visit_used_to_compute_followup_date = models.ForeignKey( "web.Visit", @@ -90,7 +111,9 @@ class StudySubject(models.Model): on_delete=models.SET_NULL, ) - screening_number = models.CharField(max_length=50, verbose_name="Screening number", blank=True, null=True) + screening_number = models.CharField( + max_length=50, verbose_name="Screening number", blank=True, null=True + ) nd_number = models.CharField( max_length=25, blank=True, @@ -98,7 +121,9 @@ class StudySubject(models.Model): ) comments = models.TextField(max_length=2000, blank=True, verbose_name="Comments") date_added = models.DateField(verbose_name="Added on", auto_now_add=True) - referral = models.CharField(max_length=128, null=True, blank=True, verbose_name="Referred by") + referral = models.CharField( + max_length=128, null=True, blank=True, verbose_name="Referred by" + ) referral_letter = models.FileField( storage=FILE_STORAGE, upload_to="referral_letters", @@ -108,7 +133,11 @@ class StudySubject(models.Model): ) health_partner = models.ForeignKey( - "web.Worker", verbose_name="Health partner", null=True, blank=True, on_delete=models.CASCADE + "web.Worker", + verbose_name="Health partner", + null=True, + blank=True, + on_delete=models.CASCADE, ) health_partner_feedback_agreement = models.BooleanField( @@ -116,16 +145,32 @@ class StudySubject(models.Model): default=False, ) - voucher_types = models.ManyToManyField(VoucherType, blank=True, verbose_name="Voucher types") + voucher_types = models.ManyToManyField( + VoucherType, blank=True, verbose_name="Voucher types" + ) - information_sent = models.BooleanField(verbose_name="Information sent", default=False) + information_sent = models.BooleanField( + verbose_name="Information sent", default=False + ) - resigned = models.BooleanField(verbose_name="Resigned", default=False, editable=True) - resign_reason = models.TextField(max_length=2000, blank=True, verbose_name="Resign reason") - excluded = models.BooleanField(verbose_name="Excluded", default=False, editable=True) - exclude_reason = models.TextField(max_length=2000, blank=True, verbose_name="Exclude reason") - endpoint_reached = models.BooleanField(verbose_name="Endpoint Reached", default=False, editable=True) - endpoint_reached_reason = models.TextField(max_length=2000, blank=True, verbose_name="Endpoint reached comments") + resigned = models.BooleanField( + verbose_name="Resigned", default=False, editable=True + ) + resign_reason = models.TextField( + max_length=2000, blank=True, verbose_name="Resign reason" + ) + excluded = models.BooleanField( + verbose_name="Excluded", default=False, editable=True + ) + exclude_reason = models.TextField( + max_length=2000, blank=True, verbose_name="Exclude reason" + ) + endpoint_reached = models.BooleanField( + verbose_name="Endpoint Reached", default=False, editable=True + ) + endpoint_reached_reason = models.TextField( + max_length=2000, blank=True, verbose_name="Endpoint reached comments" + ) def sort_matched_screening_first(self, pattern, reverse=False): if self.screening_number is None: @@ -141,7 +186,9 @@ class StudySubject(models.Model): letter, number = chunks try: tupl = (letter, int(number)) - except ValueError: # better than isdigit because isdigit fails with negative numbers and others + except ( + ValueError + ): # better than isdigit because isdigit fails with negative numbers and others tupl = (letter, number) else: logger.warning( @@ -175,7 +222,9 @@ class StudySubject(models.Model): return True def can_schedule(self): - return not any([self.resigned, self.excluded, self.endpoint_reached, self.subject.dead]) + return not any( + [self.resigned, self.excluded, self.endpoint_reached, self.subject.dead] + ) @property def status(self): @@ -192,17 +241,25 @@ class StudySubject(models.Model): @property def custom_data_values(self): - values = CustomStudySubjectValue.objects.filter(study_subject=self) - fields = list(CustomStudySubjectField.objects.filter(study=self.study)) - for value in values: - fields.remove(value.study_subject_field) + # find the custom fields that have not yet been populated into the study subject + # https://docs.djangoproject.com/en/3.2/ref/models/querysets/#filteredrelation-objects + fields = CustomStudySubjectField.objects.annotate( + t=FilteredRelation( + "customstudysubjectvalue", + condition=Q(customstudysubjectvalue__study_subject=self), + ) + ).filter(t__study_subject_field__isnull=True, study=self.study) + for field in fields: CustomStudySubjectValue.objects.create( study_subject=self, value=field.default_value, study_subject_field=field ) + return CustomStudySubjectValue.objects.filter(study_subject=self) - def set_custom_data_value(self, custom_study_subject_field: CustomStudySubjectField, value: str): + def set_custom_data_value( + self, custom_study_subject_field: CustomStudySubjectField, value: str + ): found = False for existing_value in self.customstudysubjectvalue_set.all(): if existing_value.study_subject_field == custom_study_subject_field: @@ -212,7 +269,9 @@ class StudySubject(models.Model): if not found: self.customstudysubjectvalue_set.add( CustomStudySubjectValue.objects.create( - study_subject=self, value=value, study_subject_field=custom_study_subject_field + study_subject=self, + value=value, + study_subject_field=custom_study_subject_field, ) ) @@ -220,7 +279,9 @@ class StudySubject(models.Model): # pylint: disable-next=C0209 return "%s %s" % (self.subject.first_name, self.subject.last_name) - def get_custom_data_value(self, custom_field: CustomStudySubjectField) -> Optional[CustomStudySubjectValue]: + def get_custom_data_value( + self, custom_field: CustomStudySubjectField + ) -> Optional[CustomStudySubjectValue]: for value in self.custom_data_values: if value.study_subject_field == custom_field: return value @@ -241,7 +302,9 @@ class StudySubject(models.Model): # SIGNALS @receiver(post_save, sender=StudySubject) -def set_as_resigned_or_excluded_or_endpoint_reached(sender, instance, **kwargs): # pylint: disable=unused-argument +def set_as_resigned_or_excluded_or_endpoint_reached( + sender, instance, **kwargs +): # pylint: disable=unused-argument if instance.excluded: instance.mark_as_excluded() if instance.resigned: diff --git a/smash/web/tests/view/test_export.py b/smash/web/tests/view/test_export.py index 79b82a55b392958ead903d015cd78ff5c5333750..a117e8f628671c074a925fd033deb2b172dc8e23 100644 --- a/smash/web/tests/view/test_export.py +++ b/smash/web/tests/view/test_export.py @@ -8,7 +8,7 @@ from web.models.custom_data.custom_study_subject_field import get_study_subject_ from web.tests import LoggedInTestCase from web.tests.functions import create_study_subject, create_appointment, create_visit, create_appointment_type, \ get_test_study, create_language -from web.views.export import subject_to_row_for_fields, DROP_OUT_FIELD, get_subjects_as_array +from web.views.export import subject_to_row_for_fields_processor, DROP_OUT_FIELD, get_subjects_as_array class TestExportView(LoggedInTestCase): @@ -73,7 +73,9 @@ class TestExportView(LoggedInTestCase): subject.resigned = False subject.save() - result = subject_to_row_for_fields(subject, [DROP_OUT_FIELD]) + subject2row = subject_to_row_for_fields_processor(subject.study, [DROP_OUT_FIELD]) + result = subject2row.subject_to_row_for_fields(subject) + self.assertFalse(result[0]) def test_subject_to_row_for_fields_when_resigned(self): @@ -81,7 +83,9 @@ class TestExportView(LoggedInTestCase): subject.resigned = True subject.save() - result = subject_to_row_for_fields(subject, [DROP_OUT_FIELD]) + subject2row = subject_to_row_for_fields_processor(subject.study, [DROP_OUT_FIELD]) + result = subject2row.subject_to_row_for_fields(subject) + self.assertFalse(result[0]) def test_subject_to_row_for_fields_when_dropped_out(self): @@ -94,7 +98,8 @@ class TestExportView(LoggedInTestCase): appointment.status = Appointment.APPOINTMENT_STATUS_FINISHED appointment.save() - result = subject_to_row_for_fields(subject, [DROP_OUT_FIELD]) + subject2row = subject_to_row_for_fields_processor(subject.study, [DROP_OUT_FIELD]) + result = subject2row.subject_to_row_for_fields(subject) self.assertTrue(result[0]) def test_subject_with_custom_field(self): diff --git a/smash/web/views/export.py b/smash/web/views/export.py index 379d19db0ba3ee92645488af92e7974df16f438f..841059687a354961934cecee8b9bb955840b1d72 100644 --- a/smash/web/views/export.py +++ b/smash/web/views/export.py @@ -15,14 +15,19 @@ from .view_utils import wrap_response, e500_error from ..utils import get_client_ip -@PermissionDecorator('export_subjects', 'subject') +@PermissionDecorator("export_subjects", "subject") def export_to_csv(request, study_id, data_type="subjects"): study = get_object_or_404(Study, id=study_id) # Create the HttpResponse object with the appropriate CSV header. - selected_fields = request.GET.get('fields', None) - response = HttpResponse(content_type='text/csv; charset=utf-8') - response['Content-Disposition'] = 'attachment; filename="' + data_type + '-' + get_today_midnight_date().strftime( - "%Y-%m-%d") + '.csv"' + selected_fields = request.GET.get("fields", None) + response = HttpResponse(content_type="text/csv; charset=utf-8") + response["Content-Disposition"] = ( + 'attachment; filename="' + + data_type + + "-" + + get_today_midnight_date().strftime("%Y-%m-%d") + + '.csv"' + ) if data_type == "subjects": data = get_subjects_as_array(study, selected_fields=selected_fields) @@ -38,20 +43,21 @@ def export_to_csv(request, study_id, data_type="subjects"): ip = get_client_ip(request) p = Provenance( modification_author=worker, - modification_description=f'Export {data_type} to csv', - modified_field='', + modification_description=f"Export {data_type} to csv", + modified_field="", request_path=request.path, - request_ip_addr=ip) + request_ip_addr=ip, + ) p.save() return response -@PermissionDecorator('export_subjects', 'subject') +@PermissionDecorator("export_subjects", "subject") def export_to_excel(request, study_id, data_type="subjects"): study = get_object_or_404(Study, id=study_id) - selected_fields = request.GET.get('fields', None) - filename = data_type + '-' + get_today_midnight_date().strftime("%Y-%m-%d") + ".xls" + selected_fields = request.GET.get("fields", None) + filename = data_type + "-" + get_today_midnight_date().strftime("%Y-%m-%d") + ".xls" if data_type == "subjects": data = get_subjects_as_array(study, selected_fields=selected_fields) elif data_type == "appointments": @@ -59,58 +65,51 @@ def export_to_excel(request, study_id, data_type="subjects"): else: return e500_error(request) - response = excel.make_response_from_array(data, 'xls', file_name=filename) - response['Content-Disposition'] = 'attachment; filename="' + filename + '"' + response = excel.make_response_from_array(data, "xls", file_name=filename) + response["Content-Disposition"] = 'attachment; filename="' + filename + '"' worker = Worker.get_by_user(request.user) ip = get_client_ip(request) - p = Provenance(modification_author=worker, - modification_description=f'Export {data_type} to excel', - modified_field='', - request_path=request.path, - request_ip_addr=ip) + p = Provenance( + modification_author=worker, + modification_description=f"Export {data_type} to excel", + modified_field="", + request_path=request.path, + request_ip_addr=ip, + ) p.save() return response class CustomField: - name = '' - verbose_name = '' + name = "" + verbose_name = "" def __init__(self, dictionary): for k, v in list(dictionary.items()): setattr(self, k, v) -DROP_OUT_FIELD = CustomField({'verbose_name': "DROP OUT", 'name': "custom-drop-out"}) -APPOINTMENT_TYPE_FIELD = CustomField({ - 'name': 'appointment_types', - 'verbose_name': 'Appointment Types' -}) -STUDY_SUBJECT_FIELDS = [CustomField({ - 'name': 'nd_number', - 'verbose_name': 'Subject number' -})] - -SUBJECT_FIELDS = [CustomField({ - 'name': 'last_name', - 'verbose_name': 'Family name' -}), - CustomField({ - 'name': 'first_name', - 'verbose_name': 'Name' - })] -VISIT_FIELDS = [CustomField({ - 'name': 'visit_number', - 'verbose_name': 'Visit' -})] +DROP_OUT_FIELD = CustomField({"verbose_name": "DROP OUT", "name": "custom-drop-out"}) +APPOINTMENT_TYPE_FIELD = CustomField( + {"name": "appointment_types", "verbose_name": "Appointment Types"} +) +STUDY_SUBJECT_FIELDS = [ + CustomField({"name": "nd_number", "verbose_name": "Subject number"}) +] + +SUBJECT_FIELDS = [ + CustomField({"name": "last_name", "verbose_name": "Family name"}), + CustomField({"name": "first_name", "verbose_name": "Name"}), +] +VISIT_FIELDS = [CustomField({"name": "visit_number", "verbose_name": "Visit"})] def filter_fields_from_selected_fields(fields, selected_fields): if selected_fields is None: return fields - selected_fields = set(selected_fields.split(',')) + selected_fields = set(selected_fields.split(",")) return [field for field in fields if field.name in selected_fields] @@ -127,7 +126,13 @@ def get_default_subject_fields(study: Study): subject_fields.append(DROP_OUT_FIELD) for custom_field in CustomStudySubjectField.objects.filter(study=study).all(): subject_fields.append( - CustomField({'verbose_name': custom_field.name, 'name': get_study_subject_field_id(custom_field)})) + CustomField( + { + "verbose_name": custom_field.name, + "name": get_study_subject_field_id(custom_field), + } + ) + ) return subject_fields @@ -139,60 +144,102 @@ def get_subjects_as_array(study: Study, selected_fields: str = None): field_names = [field.verbose_name for field in subject_fields] # faster than loop result.append(field_names) - subjects = StudySubject.objects.order_by('-subject__last_name') + subjects = StudySubject.objects.order_by("-subject__last_name") + subject2row = subject_to_row_for_fields_processor(study, subject_fields) for subject in subjects: - row = subject_to_row_for_fields(subject, subject_fields) + row = subject2row.subject_to_row_for_fields(subject) result.append([str(s).replace("\n", ";").replace("\r", ";") for s in row]) return result -def subject_to_row_for_fields(study_subject: StudySubject, subject_fields): - row = [] - custom_fields = CustomStudySubjectField.objects.filter(study=study_subject.study).all() +class subject_to_row_for_fields_processor: + def __init__(self, study, subject_fields): + self.custom_fields_map = self._get_custom_fields_map(study) + self.subject_fields = subject_fields + self.study_subject_fields = set() # emtpy + self.study_subject_subject_fields = set() # emtpy - for field in subject_fields: - cell = None + # pylint: disable=R6301 + def _get_custom_fields_map(self, study): + custom_fields = CustomStudySubjectField.objects.filter(study=study).all() + custom_fields_map = {} for custom_field in custom_fields: - if get_study_subject_field_id(custom_field) == field.name: - val = study_subject.get_custom_data_value(custom_field) - if val is not None: - cell = val.value - - if field == DROP_OUT_FIELD: - if not study_subject.resigned: - cell = False - else: - finished_appointments = Appointment.objects.filter(visit__subject=study_subject).filter( - status=Appointment.APPOINTMENT_STATUS_FINISHED).count() - if finished_appointments > 0: - cell = True - else: + custom_fields_map[get_study_subject_field_id(custom_field)] = custom_field + return custom_fields_map + + def subject_to_row_for_fields(self, study_subject: StudySubject): + + # cache fields + if len(self.study_subject_fields) == 0: + for field in self.subject_fields: + if hasattr(study_subject, field.name): + self.study_subject_fields.add(field.name) + elif hasattr(study_subject.subject, field.name): + self.study_subject_subject_fields.add(field.name) + + # to avoid re-execution of this function (property) inside get_custom_data_value + custom_data_values = study_subject.custom_data_values + custom_data_values_map = { + value.study_subject_field: value for value in custom_data_values + } # considerably faster than for loop + + row = [] + + for field in self.subject_fields: + cell = None + + if field.name in self.custom_fields_map: + custom_field = self.custom_fields_map[field.name] + if custom_field in custom_data_values_map: + cell = custom_data_values_map[custom_field].value + + if field == DROP_OUT_FIELD: + if not study_subject.resigned: cell = False - else: - if hasattr(study_subject, field.name): - cell = getattr(study_subject, field.name) - elif hasattr(study_subject.subject, field.name): - cell = getattr(study_subject.subject, field.name) - if cell is None: - cell = "" - if isinstance(cell, BaseManager): - collection_value = "" - for value in cell.all(): - collection_value += str(value) + "," - cell = collection_value - row.append(cell) - return row + else: + finished_appointments = ( + Appointment.objects.filter(visit__subject=study_subject) + .filter(status=Appointment.APPOINTMENT_STATUS_FINISHED) + .count() + ) + cell = finished_appointments > 0 + else: + if field.name in self.study_subject_fields: + cell = getattr(study_subject, field.name) + elif field.name in self.study_subject_subject_fields: + cell = getattr(study_subject.subject, field.name) + if cell is None: + cell = "" + if isinstance(cell, BaseManager): + collection_value = "" + for value in cell.all(): + collection_value += str(value) + "," + cell = collection_value + row.append(cell) + + return row def get_appointment_fields(): appointments_fields = [] for field in Appointment._meta.fields: - if field.name.upper() != "VISIT" and field.name.upper() != "ID" and \ - field.name.upper() != "WORKER_ASSIGNED" and field.name.upper() != "APPOINTMENT_TYPES" and \ - field.name.upper() != "ROOM" and field.name.upper() != "FLYING_TEAM": + if ( + field.name.upper() != "VISIT" + and field.name.upper() != "ID" + and field.name.upper() != "WORKER_ASSIGNED" + and field.name.upper() != "APPOINTMENT_TYPES" + and field.name.upper() != "ROOM" + and field.name.upper() != "FLYING_TEAM" + ): appointments_fields.append(field) - all_fields = STUDY_SUBJECT_FIELDS + SUBJECT_FIELDS + VISIT_FIELDS + appointments_fields + [APPOINTMENT_TYPE_FIELD] + all_fields = ( + STUDY_SUBJECT_FIELDS + + SUBJECT_FIELDS + + VISIT_FIELDS + + appointments_fields + + [APPOINTMENT_TYPE_FIELD] + ) return all_fields, appointments_fields @@ -201,12 +248,14 @@ def get_appointments_as_array(selected_fields=None): result = [] all_fields, appointments_fields = get_appointment_fields() all_fields = filter_fields_from_selected_fields(all_fields, selected_fields) - appointments_fields = filter_fields_from_selected_fields(appointments_fields, selected_fields) + appointments_fields = filter_fields_from_selected_fields( + appointments_fields, selected_fields + ) field_names = [field.verbose_name for field in all_fields] # faster than loop result.append(field_names) - appointments = Appointment.objects.order_by('-datetime_when') + appointments = Appointment.objects.order_by("-datetime_when") for appointment in appointments: # add field_names ['ND number', 'Family name', 'Name', 'Visit'] first @@ -214,36 +263,45 @@ def get_appointments_as_array(selected_fields=None): for field in STUDY_SUBJECT_FIELDS: if field.verbose_name in field_names: if appointment.visit is None: - row.append('---') + row.append("---") else: row.append(getattr(appointment.visit.subject, field.name)) for field in SUBJECT_FIELDS: if field.verbose_name in field_names: if appointment.visit is None: - row.append('---') + row.append("---") else: row.append(getattr(appointment.visit.subject.subject, field.name)) for field in VISIT_FIELDS: if field.verbose_name in field_names: if appointment.visit is None: - row.append('---') + row.append("---") else: row.append(getattr(appointment.visit, field.name)) for field in appointments_fields: row.append(getattr(appointment, field.name)) if APPOINTMENT_TYPE_FIELD.verbose_name in field_names: # avoid last comma in the list of appointment types - type_string = ','.join([appointment_type.code for appointment_type in appointment.appointment_types.all()]) + type_string = ",".join( + [ + appointment_type.code + for appointment_type in appointment.appointment_types.all() + ] + ) row.append(type_string) result.append([str(s).replace("\n", ";").replace("\r", ";") for s in row]) return result -@PermissionDecorator('export_subjects', 'subject') +@PermissionDecorator("export_subjects", "subject") def export(request, study_id): study = get_object_or_404(Study, id=study_id) - return wrap_response(request, 'export/index.html', { - 'subject_fields': get_default_subject_fields(study), - 'appointment_fields': get_appointment_fields()[0], - 'study_id': study_id - }) + return wrap_response( + request, + "export/index.html", + { + "subject_fields": get_default_subject_fields(study), + "appointment_fields": get_appointment_fields()[0], + "study_id": study_id, + }, + )