Resolve "configurable study fields"

Closes #339 (closed)

@carlos.vega

The changes can be summorized as follow:

• study configuration allows to add custom field to StudySubject.
• every StudySubject has set of values for custom fields in the study
• custom field can be of following type: text, integer, double, boolean, date, select from predefined list, file
• StudySubject files are uploaded to separate folder defined in local_settings.py - this folder should not be accessed by WhiteNoise/nginx/apache - there is upload_files view that allows to download content of the file if user is authenticated
• custom field can be marked as read-only, obligatory, unique (in a study)
• custom field can have default value defined
• urls for adding new subject, exporting subjects, fetching data about subject from API contain study_id - this was required for newly implemented tests

smash/web/api_views/subject.py

 import logging
 import re
+from distutils.util import strtobool
 
-from django.db.models import Count, Case, When, Min, Max
+from django.db.models import Count, Case, When, Min, Max, QuerySet
 from django.db.models import Q
 from django.http import JsonResponse
 from django.urls import reverse
-from web.models import ConfigurationItem
-from distutils.util import strtobool
 
-from web.api_views.serialization_utils import bool_to_yes_no, flying_team_to_str, location_to_str, add_column, \
-    serialize_date, serialize_datetime, get_filters_for_data_table_request, virus_test_to_str
-from web.models import StudySubject, Visit, Appointment, Subject, SubjectColumns, StudyColumns, Study, ContactAttempt
-from web.models.constants import SUBJECT_TYPE_CHOICES, GLOBAL_STUDY_ID, VISIT_SHOW_VISIT_NUMBER_FROM_ZERO
+from web.api_views.serialization_utils import str_to_yes_no_null, bool_to_yes_no, flying_team_to_str, location_to_str, add_column, \
+    serialize_date, serialize_datetime, get_filters_for_data_table_request, virus_test_to_str, str_to_yes_no
+from web.models import ConfigurationItem, StudySubject, Visit, Appointment, Subject, SubjectColumns, StudyColumns, \
+    Study, ContactAttempt
+from web.models.constants import SUBJECT_TYPE_CHOICES, GLOBAL_STUDY_ID, VISIT_SHOW_VISIT_NUMBER_FROM_ZERO, \
+    CUSTOM_FIELD_TYPE_TEXT, CUSTOM_FIELD_TYPE_BOOLEAN, CUSTOM_FIELD_TYPE_INTEGER, CUSTOM_FIELD_TYPE_DOUBLE, \
+    CUSTOM_FIELD_TYPE_DATE, CUSTOM_FIELD_TYPE_SELECT_LIST, CUSTOM_FIELD_TYPE_FILE
+from web.models.custom_data.custom_study_subject_field import get_study_subject_field_id, CustomStudySubjectField
 from web.models.study_subject_list import SUBJECT_LIST_GENERIC, SUBJECT_LIST_NO_VISIT, SUBJECT_LIST_REQUIRE_CONTACT, \
     StudySubjectList, SUBJECT_LIST_VOUCHER_EXPIRY
 from web.views import e500_error
@@ -37,6 +40,7 @@ def referrals(request):
     })
 
 
+# noinspection PyUnusedLocal
 def get_subject_columns(request, subject_list_type):
     study = Study.objects.filter(id=GLOBAL_STUDY_ID)[0]
     study_subject_lists = StudySubjectList.objects.filter(study=study, type=subject_list_type)
@@ -118,6 +122,61 @@ def get_subject_columns(request, subject_list_type):
                    'serology_filter', study.columns)
 
     add_column(result, "Type", "type", study_subject_columns, "type_filter", study.columns)
+    for custom_study_subject_field in study.customstudysubjectfield_set.all():
+        if custom_study_subject_field.type == CUSTOM_FIELD_TYPE_TEXT:
+            add_column(result,
+                       custom_study_subject_field.name,
+                       get_study_subject_field_id(custom_study_subject_field),
+                       study_subject_columns,
+                       "string_filter",
+                       visible_param=False)
+        elif custom_study_subject_field.type == CUSTOM_FIELD_TYPE_BOOLEAN:
+            add_column(result,
+                       custom_study_subject_field.name,
+                       get_study_subject_field_id(custom_study_subject_field),
+                       study_subject_columns,
+                       "yes_no_filter",
+                       visible_param=False)
+        elif custom_study_subject_field.type == CUSTOM_FIELD_TYPE_INTEGER:
+            add_column(result,
+                       custom_study_subject_field.name,
+                       get_study_subject_field_id(custom_study_subject_field),
+                       study_subject_columns,
+                       None,
+                       sortable=False,
+                       visible_param=False)
+        elif custom_study_subject_field.type == CUSTOM_FIELD_TYPE_DOUBLE:
+            add_column(result,
+                       custom_study_subject_field.name,
+                       get_study_subject_field_id(custom_study_subject_field),
+                       study_subject_columns,
+                       None,
+                       sortable=False,
+                       visible_param=False)
+        elif custom_study_subject_field.type == CUSTOM_FIELD_TYPE_DATE:
+            add_column(result,
+                       custom_study_subject_field.name,
+                       get_study_subject_field_id(custom_study_subject_field),
+                       study_subject_columns,
+                       None,
+                       visible_param=False)
+        elif custom_study_subject_field.type == CUSTOM_FIELD_TYPE_SELECT_LIST:
+            add_column(result,
+                       custom_study_subject_field.name,
+                       get_study_subject_field_id(custom_study_subject_field),
+                       study_subject_columns,
+                       'select_filter:' + custom_study_subject_field.possible_values,
+                       visible_param=False)
+        elif custom_study_subject_field.type == CUSTOM_FIELD_TYPE_FILE:
+            add_column(result,
+                       custom_study_subject_field.name,
+                       get_study_subject_field_id(custom_study_subject_field),
+                       study_subject_columns,
+                       'select_filter:N/A;Available',
+                       visible_param=False)
+        else:
+            raise NotImplementedError
+
     add_column(result, "Edit", "edit", None, None, sortable=False)
 
     for one_based_idx, visit_number in enumerate(visit_numbers, 1):
@@ -128,17 +187,17 @@ def get_subject_columns(request, subject_list_type):
     return JsonResponse({"columns": result})
 
 
-def get_subjects(request, type):
-    if type == SUBJECT_LIST_GENERIC:
+def get_subjects(request, list_type):
+    if list_type == SUBJECT_LIST_GENERIC:
         return StudySubject.objects.all()
-    elif type == SUBJECT_LIST_NO_VISIT:
+    elif list_type == SUBJECT_LIST_NO_VISIT:
         return get_subjects_with_no_visit(request.user)
-    elif type == SUBJECT_LIST_REQUIRE_CONTACT:
+    elif list_type == SUBJECT_LIST_REQUIRE_CONTACT:
         return get_subjects_with_reminder(request.user)
-    elif type == SUBJECT_LIST_VOUCHER_EXPIRY:
+    elif list_type == SUBJECT_LIST_VOUCHER_EXPIRY:
         return get_subjects_with_almost_expired_vouchers(request.user)
     else:
-        raise TypeError("Unknown query type: " + type)
+        raise TypeError("Unknown query type: " + list_type)
 
 
 def order_by_visit(subjects_to_be_ordered, order_direction, visit_number):
@@ -147,14 +206,16 @@ def order_by_visit(subjects_to_be_ordered, order_direction, visit_number):
         order_direction + 'sort_visit_date')
 
 
-def get_subjects_order(subjects_to_be_ordered, order_column, order_direction, column_filters={}):
+def get_subjects_order(subjects_to_be_ordered: QuerySet, order_column, order_direction, column_filters=None):
+    if column_filters is None:
+        column_filters = {}
     result = subjects_to_be_ordered
     if order_direction == "asc":
         order_direction = ""
     else:
         order_direction = "-"
     if order_column is None:
-        logger.warn("Column cannot be null")
+        logger.warning("Column cannot be null")
     elif order_column == "first_name":
         result = subjects_to_be_ordered.order_by(order_direction + 'subject__first_name')
     elif order_column == "last_name":
@@ -228,8 +289,14 @@ def get_subjects_order(subjects_to_be_ordered, order_column, order_direction, co
         result = subjects_to_be_ordered.order_by(order_direction + order_column)
     elif re.search(r'^virus_test_[1-5]_igg_status', order_column):
         result = subjects_to_be_ordered.order_by(order_direction + order_column)
+    elif re.search(r'^custom_field-[0-9]+$', order_column):
+        field_id = int(order_column.replace("custom_field-", ""))
+        result = subjects_to_be_ordered.annotate(
+            custom_field_value=Min(Case(When(customstudysubjectvalue__study_subject_field__id=field_id,
+                                             then='customstudysubjectvalue__value')))).order_by(
+            order_direction + 'custom_field_value')
     else:
-        logger.warn("Unknown sort column: " + str(order_column))
+        logger.warning("Unknown sort column: " + str(order_column))
     return result
 
 
@@ -298,13 +365,13 @@ def filter_by_visit(result, visit_number, visit_type):
     return result
 
 
-def get_subjects_filtered(subjects_to_be_filtered, filters):
+def get_subjects_filtered(subjects_to_be_filtered: QuerySet, filters) -> QuerySet:
     result = subjects_to_be_filtered
     for row in filters:
         column = row[0]
         value = row[1]
         if column is None:
-            logger.warn("Filter column cannot be null")
+            logger.warning("Filter column cannot be null")
         elif column == "first_name":
             result = result.filter(subject__first_name__icontains=value)
         elif column == "last_name":
@@ -371,6 +438,33 @@ def get_subjects_filtered(subjects_to_be_filtered, filters):
         elif str(column).startswith("visit_"):
             visit_number = get_visit_number_from_visit_x_string(column)
             result = filter_by_visit(result, visit_number, value)
+        elif re.search(r'^custom_field-[0-9]+$', column):
+            field_id = int(column.replace("custom_field-", ""))
+            field = CustomStudySubjectField.objects.get(pk=field_id)
+            if field.type == CUSTOM_FIELD_TYPE_TEXT:
+                result = result.filter(customstudysubjectvalue__study_subject_field__id=field_id,
+                                       customstudysubjectvalue__value__icontains=value)
+            elif field.type == CUSTOM_FIELD_TYPE_BOOLEAN:
+                if value.lower() == 'true' or value.lower() == 'false':
+                    result = result.filter(customstudysubjectvalue__study_subject_field__id=field_id,
+                                           customstudysubjectvalue__value__icontains=value)
+                else:
+                    result = result.filter(customstudysubjectvalue__study_subject_field__id=field_id,
+                                           customstudysubjectvalue__value='')
+            elif field.type == CUSTOM_FIELD_TYPE_INTEGER or field.type == CUSTOM_FIELD_TYPE_DOUBLE:
+                result = result.filter(customstudysubjectvalue__study_subject_field__id=field_id,
+                                       customstudysubjectvalue__value=value)
+            elif field.type == CUSTOM_FIELD_TYPE_DATE:
+                result = result.filter(customstudysubjectvalue__study_subject_field__id=field_id,
+                                       customstudysubjectvalue__value=value)
+            elif field.type == CUSTOM_FIELD_TYPE_INTEGER or field.type == CUSTOM_FIELD_TYPE_SELECT_LIST:
+                result = result.filter(customstudysubjectvalue__study_subject_field__id=field_id,
+                                       customstudysubjectvalue__value=value)
+            elif field.type == CUSTOM_FIELD_TYPE_FILE:
+                result = result.filter(customstudysubjectvalue__study_subject_field__id=field_id,
+                                       customstudysubjectvalue__value__isnull=(value == 'N/A'))
+            else:
+                raise NotImplementedError
         elif column == "":
             pass
         else:
@@ -379,11 +473,11 @@ def get_subjects_filtered(subjects_to_be_filtered, filters):
                 message += "[None]"
             else:
                 message += str(column)
-            logger.warn(message)
+            logger.warning(message)
     return result
 
 
-def subjects(request, type):
+def subjects(request, subject_list_type):
     try:
         # id of the query from dataTable: https://datatables.net/manual/server-side
         draw = int(request.GET.get("draw", "-1"))
@@ -396,7 +490,7 @@ def subjects(request, type):
 
         filters = get_filters_for_data_table_request(request)
 
-        all_subjects = get_subjects(request, type)
+        all_subjects = get_subjects(request, subject_list_type)
 
         count = all_subjects.count()
 
@@ -532,4 +626,28 @@ def serialize_subject(study_subject):
                                                                     "virus_test_{}_collection_date".format(i))
         result["virus_test_{}_iga_status".format(i)] = getattr(study_subject, "virus_test_{}_iga_status".format(i))
         result["virus_test_{}_igg_status".format(i)] = getattr(study_subject, "virus_test_{}_igg_status".format(i))
+
+    for field_value in study_subject.custom_data_values:
+        if field_value.study_subject_field.type == CUSTOM_FIELD_TYPE_TEXT \
+                or field_value.study_subject_field.type == CUSTOM_FIELD_TYPE_INTEGER \
+                or field_value.study_subject_field.type == CUSTOM_FIELD_TYPE_DOUBLE:
+            val = field_value.value
+            if val is None:
+                val = ''
+            result[get_study_subject_field_id(field_value.study_subject_field)] = val
+        elif field_value.study_subject_field.type == CUSTOM_FIELD_TYPE_BOOLEAN:
+            result[get_study_subject_field_id(field_value.study_subject_field)] = str_to_yes_no_null(field_value.value)
+        elif field_value.study_subject_field.type == CUSTOM_FIELD_TYPE_DATE:
+            result[get_study_subject_field_id(field_value.study_subject_field)] = field_value.value
+        elif field_value.study_subject_field.type == CUSTOM_FIELD_TYPE_SELECT_LIST:
+            result[get_study_subject_field_id(field_value.study_subject_field)] = field_value.value
+        elif field_value.study_subject_field.type == CUSTOM_FIELD_TYPE_FILE:
+            if field_value.value is None:
+                result[get_study_subject_field_id(field_value.study_subject_field)] = ''
+            else:
+                result[get_study_subject_field_id(field_value.study_subject_field)] = reverse(
+                    'web.views.uploaded_files') + '?file=' + str(field_value.value)
+        else:
+            raise NotImplementedError
+
     return result