W1514 lint encoding when opening files. utf-8

smash/web/importer/csv_subject_exporter.py

@@ -10,15 +10,15 @@ from ..models.etl.subject_export import SubjectExportData
 logger = logging.getLogger(__name__)
 
 
-class CsvSubjectExporter():
+class CsvSubjectExporter:
     def __init__(self, export_data: SubjectExportData):
         self.export_data = export_data
 
     def execute(self):
-        with open(self.export_data.get_absolute_file_path(), 'w') as csvfile:
+        with open(self.export_data.get_absolute_file_path(), "w", encoding="utf-8") as csvfile:
             writer = csv.writer(csvfile, delimiter=self.export_data.csv_delimiter, quoting=csv.QUOTE_ALL)
             writer.writerow(self.create_header())
-            subjects = StudySubject.objects.order_by('-id')
+            subjects = StudySubject.objects.order_by("-id")
             for subject in subjects:
                 row = self.subject_to_row(subject)
                 writer.writerow(row)
@@ -26,7 +26,7 @@ class CsvSubjectExporter():
     def subject_to_row(self, study_subject: StudySubject):
         result = []
         for mapping in self.export_data.column_mappings.order_by("id"):
-            cell = ''
+            cell = ""
             for field in StudySubject._meta.get_fields():
                 if field_can_be_exported(field):
                     if mapping.table_name == StudySubject._meta.db_table and field.name == mapping.column_name:
@@ -39,7 +39,8 @@ class CsvSubjectExporter():
 
             if mapping.table_name == CustomStudySubjectField._meta.db_table:
                 cell = study_subject.get_custom_data_value(
-                    CustomStudySubjectField.objects.get(pk=int(mapping.column_name.replace("custom_field-", '')))).value
+                    CustomStudySubjectField.objects.get(pk=int(mapping.column_name.replace("custom_field-", "")))
+                ).value
 
             result.append(cell)
         return result

smash/web/importer/csv_subject_import_reader.py

@@ -25,9 +25,10 @@ class CsvSubjectImportReader(SubjectImportReader):
 
     def load_data(self) -> List[StudySubject]:
         study_subjects = []
-        with open(self.import_data.get_absolute_file_path()) as csv_file:
-            reader = csv.reader((EtlCommon.remove_bom(line) for line in csv_file),
-                                delimiter=self.import_data.csv_delimiter)
+        with open(self.import_data.get_absolute_file_path(), encoding="utf-8") as csv_file:
+            reader = csv.reader(
+                (EtlCommon.remove_bom(line) for line in csv_file), delimiter=self.import_data.csv_delimiter
+            )
             headers = next(reader, None)
             for header in headers:
                 field = self.get_table_and_field(header)[1]

smash/web/importer/csv_visit_exporter.py

@@ -14,10 +14,10 @@ class CsvVisitExporter:
         self.export_data = export_data
 
     def execute(self):
-        with open(self.export_data.get_absolute_file_path(), 'w') as csvfile:
+        with open(self.export_data.get_absolute_file_path(), "w", encoding="utf-8") as csvfile:
             writer = csv.writer(csvfile, delimiter=self.export_data.csv_delimiter, quoting=csv.QUOTE_ALL)
             writer.writerow(self.create_header())
-            appointments = Appointment.objects.order_by('-id')
+            appointments = Appointment.objects.order_by("-id")
             for appointment in appointments:
                 row = self.appointment_to_row(appointment)
                 writer.writerow(row)
@@ -27,9 +27,9 @@ class CsvVisitExporter:
         if appointment.visit is not None and appointment.visit.subject is not None:
             result.append(appointment.visit.subject.nd_number)
         else:
-            result.append('N/A')
+            result.append("N/A")
         for mapping in self.export_data.column_mappings.order_by("id"):
-            cell = ''
+            cell = ""
             for field in Appointment._meta.get_fields():
                 if field_can_be_exported(field):
                     if mapping.table_name == Appointment._meta.db_table and field.name == mapping.column_name:

smash/web/importer/csv_visit_import_reader.py

@@ -33,12 +33,13 @@ class CsvVisitImportReader(EtlCommon):
     def load_data(self):
         filename = self.import_data.get_absolute_file_path()
         warning_counter = MsgCounterHandler()
-        logging.getLogger('').addHandler(warning_counter)
+        logging.getLogger("").addHandler(warning_counter)
 
         result = []
-        with open(filename) as csv_file:
-            reader = csv.reader((EtlCommon.remove_bom(line) for line in csv_file),
-                                delimiter=self.import_data.csv_delimiter)
+        with open(filename, encoding="utf-8") as csv_file:
+            reader = csv.reader(
+                (EtlCommon.remove_bom(line) for line in csv_file), delimiter=self.import_data.csv_delimiter
+            )
             headers = next(reader, None)
             for row in reader:
                 # noinspection PyBroadException
@@ -59,11 +60,17 @@ class CsvVisitImportReader(EtlCommon):
 
                     for i in range(1, visit_number, 1):
                         if Visit.objects.filter(subject=study_subject, visit_number=i).count() == 0:
-                            logger.warning("Previous visit (visit %s) for subject %s does not exist. Creating empty",
-                                           str(i), nd_number)
-                            Visit.objects.create(subject=study_subject, visit_number=i,
-                                                 datetime_begin=date + datetime.timedelta(days=-1, minutes=i),
-                                                 datetime_end=date + datetime.timedelta(days=-1, minutes=i))
+                            logger.warning(
+                                "Previous visit (visit %s) for subject %s does not exist. Creating empty",
+                                str(i),
+                                nd_number,
+                            )
+                            Visit.objects.create(
+                                subject=study_subject,
+                                visit_number=i,
+                                datetime_begin=date + datetime.timedelta(days=-1, minutes=i),
+                                datetime_end=date + datetime.timedelta(days=-1, minutes=i),
+                            )
                         else:
                             break
 
@@ -72,38 +79,44 @@ class CsvVisitImportReader(EtlCommon):
                         logger.debug("Visit for subject %s already exists. Updating", nd_number)
                         visit = visits[0]
 
-                        self.create_provenance_and_change_data(visit, 'datetime_begin', date, Visit)
-                        self.create_provenance_and_change_data(visit, 'datetime_end',
-                                                               date + datetime.timedelta(days=14),
-                                                               Visit)
+                        self.create_provenance_and_change_data(visit, "datetime_begin", date, Visit)
+                        self.create_provenance_and_change_data(
+                            visit, "datetime_end", date + datetime.timedelta(days=14), Visit
+                        )
                         visit.save()
                     else:
-                        visit = Visit.objects.create(subject=study_subject, visit_number=visit_number,
-                                                     datetime_begin=date,
-                                                     datetime_end=date + datetime.timedelta(days=14))
+                        visit = Visit.objects.create(
+                            subject=study_subject,
+                            visit_number=visit_number,
+                            datetime_begin=date,
+                            datetime_end=date + datetime.timedelta(days=14),
+                        )
                         visit.save()
                         # visit does not have id until .save() is done
                         self.create_provenance_for_new_object(Visit, visit)
 
                     result.append(visit)
 
-                    appointments = Appointment.objects.filter(visit=visit,
-                                                              appointment_types=self.import_data.appointment_type)
+                    appointments = Appointment.objects.filter(
+                        visit=visit, appointment_types=self.import_data.appointment_type
+                    )
                     if len(appointments) > 0:
                         logger.debug("Appointment for subject %s already set. Updating", nd_number)
                         appointment = appointments[0]
 
-                        self.create_provenance_and_change_data(appointment, 'length', 60, Appointment)
-                        self.create_provenance_and_change_data(appointment, 'datetime_when', date, Appointment)
-                        self.create_provenance_and_change_data(appointment, 'location', location, Appointment)
+                        self.create_provenance_and_change_data(appointment, "length", 60, Appointment)
+                        self.create_provenance_and_change_data(appointment, "datetime_when", date, Appointment)
+                        self.create_provenance_and_change_data(appointment, "location", location, Appointment)
 
                         appointment.save()
                     else:
-                        appointment = Appointment.objects.create(visit=visit, length=60, datetime_when=date,
-                                                                 location=location)
+                        appointment = Appointment.objects.create(
+                            visit=visit, length=60, datetime_when=date, location=location
+                        )
                         if self.import_data.appointment_type is not None:
-                            AppointmentTypeLink.objects.create(appointment_id=appointment.id,
-                                                               appointment_type=self.import_data.appointment_type)
+                            AppointmentTypeLink.objects.create(
+                                appointment_id=appointment.id, appointment_type=self.import_data.appointment_type
+                            )
 
                         appointment.save()
                         # appointment does not have id until .save() is done
@@ -112,11 +125,11 @@ class CsvVisitImportReader(EtlCommon):
                 except BaseException:
                     self.problematic_count += 1
                     traceback.print_exc(file=sys.stdout)
-                    logger.exception("Problematic data: %s", ';'.join(row))
+                    logger.exception("Problematic data: %s", ";".join(row))
 
         if "WARNING" in warning_counter.level2count:
             self.warning_count = warning_counter.level2count["WARNING"]
-        logging.getLogger('').removeHandler(warning_counter)
+        logging.getLogger("").removeHandler(warning_counter)
 
         return result
 
@@ -124,18 +137,20 @@ class CsvVisitImportReader(EtlCommon):
         try:
             return self.extract_date(data[self.import_data.visit_date_column_name])
         except KeyError as e:
-            raise EtlException('Visit date is not defined') from e
+            raise EtlException("Visit date is not defined") from e
 
     def get_study_subject_by_id(self, nd_number: str) -> StudySubject:
         study_subjects = StudySubject.objects.filter(nd_number=nd_number, study=self.import_data.study)
         if len(study_subjects) == 0:
             logger.debug("Subject %s does not exist. Creating", nd_number)
             subject = Subject.objects.create()
-            study_subject = StudySubject.objects.create(subject=subject,
-                                                        study=self.import_data.study,
-                                                        nd_number=nd_number,
-                                                        screening_number=nd_number,
-                                                        type=SubjectType.objects.all().first())
+            study_subject = StudySubject.objects.create(
+                subject=subject,
+                study=self.import_data.study,
+                nd_number=nd_number,
+                screening_number=nd_number,
+                type=SubjectType.objects.all().first(),
+            )
         else:
             study_subject = study_subjects[0]
         return study_subject
@@ -145,7 +160,7 @@ class CsvVisitImportReader(EtlCommon):
             nd_number = data[self.import_data.subject_id_column_name]
             return nd_number
         except KeyError as e:
-            raise EtlException('Subject id is not defined') from e
+            raise EtlException("Subject id is not defined") from e
 
     def extract_date(self, text: str) -> datetime:
         try:
@@ -167,16 +182,21 @@ class CsvVisitImportReader(EtlCommon):
                 logger.warning("Location with name does not exist: %s", text)
                 return Location.objects.create(name=text)
         except KeyError as e:
-            raise EtlException('Location is not defined') from e
+            raise EtlException("Location is not defined") from e
 
     def get_summary(self):
         result = "<p>Number of successfully added appointments: <b>" + str(self.processed_count) + "</b></p>"
-        style = ''
+        style = ""
         if self.problematic_count > 0:
             style = ' color="red" '
-        result += "<p><font " + style + ">Number of problematic appointments: <b>" + str(
-            self.problematic_count) + "</b></font></p>"
-        style = ''
+        result += (
+            "<p><font "
+            + style
+            + ">Number of problematic appointments: <b>"
+            + str(self.problematic_count)
+            + "</b></font></p>"
+        )
+        style = ""
         if self.warning_count > 0:
             style = ' color="brown" '
         result += "<p><font " + style + ">Number of raised warnings: <b>" + str(self.warning_count) + "</b></font></p>"
@@ -187,9 +207,11 @@ class CsvVisitImportReader(EtlCommon):
             visit_number = data[self.import_data.visit_number_column_name]
             visit_number = int(visit_number) + (1 - self.import_data.study.redcap_first_visit_number)
             if visit_number < 1:
-                logger.warning("Visit number is invalid. Visit number should start from: %d.",
-                               self.import_data.study.redcap_first_visit_number)
+                logger.warning(
+                    "Visit number is invalid. Visit number should start from: %d.",
+                    self.import_data.study.redcap_first_visit_number,
+                )
                 visit_number = 1
             return visit_number
         except KeyError as e:
-            raise EtlException('Visit number is not defined') from e
+            raise EtlException("Visit number is not defined") from e

smash/web/tests/importer/test_subject_exporter.py

@@ -18,12 +18,12 @@ logger = logging.getLogger(__name__)
 
 
 class TestExporter(TestCase):
-
     def setUp(self):
         self.study = get_test_study()
-        self.visit_export_data: SubjectExportData = SubjectExportData.objects.create(study=self.study,
-                                                                                     filename="test.csv")
-        self.visit_export_data.csv_delimiter = ','
+        self.visit_export_data: SubjectExportData = SubjectExportData.objects.create(
+            study=self.study, filename="test.csv"
+        )
+        self.visit_export_data.csv_delimiter = ","
 
         for field in Subject._meta.get_fields():
             if field_can_be_exported(field):
@@ -47,7 +47,7 @@ class TestExporter(TestCase):
         exporter = CsvSubjectExporter(self.visit_export_data)
         exporter.execute()
 
-        with open(self.visit_export_data.get_absolute_file_path(), 'r') as file:
+        with open(self.visit_export_data.get_absolute_file_path(), "r", encoding="utf-8") as file:
             data = file.read()
             self.assertTrue(study_subject.nd_number in data)
 
@@ -58,7 +58,7 @@ class TestExporter(TestCase):
         exporter = CsvSubjectExporter(self.visit_export_data)
         exporter.execute()
 
-        with open(self.visit_export_data.get_absolute_file_path(), 'r') as file:
+        with open(self.visit_export_data.get_absolute_file_path(), "r", encoding="utf-8") as file:
             data = file.read()
             self.assertFalse(study_subject.type.name in data)
             self.assertTrue(str(study_subject.type_id) in data)
@@ -70,28 +70,30 @@ class TestExporter(TestCase):
         exporter = CsvSubjectExporter(self.visit_export_data)
         exporter.execute()
 
-        with open(self.visit_export_data.get_absolute_file_path(), 'r') as file:
+        with open(self.visit_export_data.get_absolute_file_path(), "r", encoding="utf-8") as file:
             data = file.read()
             self.assertTrue(study_subject.type.name in data)
 
     def test_export_custom_field(self):
-        field = CustomStudySubjectField.objects.create(study=get_test_study(), name='my custom field',
-                                                       type=CUSTOM_FIELD_TYPE_TEXT)
+        field = CustomStudySubjectField.objects.create(
+            study=get_test_study(), name="my custom field", type=CUSTOM_FIELD_TYPE_TEXT
+        )
 
-        self.visit_export_data.set_column_mapping(CustomStudySubjectField, get_study_subject_field_id(field),
-                                                  'my_custom_field')
+        self.visit_export_data.set_column_mapping(
+            CustomStudySubjectField, get_study_subject_field_id(field), "my_custom_field"
+        )
 
         study_subject = create_study_subject()
-        study_subject.set_custom_data_value(field, 'blah-blah')
+        study_subject.set_custom_data_value(field, "blah-blah")
 
         self.visit_export_data.export_object_as_id = True
         exporter = CsvSubjectExporter(self.visit_export_data)
         exporter.execute()
 
-        with open(self.visit_export_data.get_absolute_file_path(), 'r') as file:
+        with open(self.visit_export_data.get_absolute_file_path(), "r", encoding="utf-8") as file:
             data = file.read()
-            self.assertTrue('my_custom_field' in data)
-            self.assertTrue('blah-blah' in data)
+            self.assertTrue("my_custom_field" in data)
+            self.assertTrue("blah-blah" in data)
 
     def test_export_languages(self):
         study_subject = create_study_subject()
@@ -105,7 +107,7 @@ class TestExporter(TestCase):
         exporter = CsvSubjectExporter(self.visit_export_data)
         exporter.execute()
 
-        with open(self.visit_export_data.get_absolute_file_path(), 'r') as file:
+        with open(self.visit_export_data.get_absolute_file_path(), "r", encoding="utf-8") as file:
             data = file.read()
             self.assertTrue("Polish" in data)
 
@@ -114,12 +116,12 @@ class TestExporter(TestCase):
         study_subject.subject.date_born = now()
         study_subject.subject.save()
 
-        self.visit_export_data.date_format = '%Y/%m/%d'
+        self.visit_export_data.date_format = "%Y/%m/%d"
         self.visit_export_data.export_object_as_id = False
 
         exporter = CsvSubjectExporter(self.visit_export_data)
         exporter.execute()
 
-        with open(self.visit_export_data.get_absolute_file_path(), 'r') as file:
+        with open(self.visit_export_data.get_absolute_file_path(), "r", encoding="utf-8") as file:
             data = file.read()
             self.assertTrue(study_subject.subject.date_born.strftime(self.visit_export_data.date_format) in data)

smash/web/tests/importer/test_visit_exporter.py

@@ -15,12 +15,12 @@ logger = logging.getLogger(__name__)
 
 
 class TestVisitExporter(TestCase):
-
     def setUp(self):
         self.study = get_test_study()
-        self.visit_export_data: VisitExportData = VisitExportData.objects.create(study=self.study,
-                                                                                 filename="visit_test.csv")
-        self.visit_export_data.csv_delimiter = ','
+        self.visit_export_data: VisitExportData = VisitExportData.objects.create(
+            study=self.study, filename="visit_test.csv"
+        )
+        self.visit_export_data.csv_delimiter = ","
 
         for field in Appointment._meta.get_fields():
             if field_can_be_exported(field):
@@ -36,36 +36,36 @@ class TestVisitExporter(TestCase):
 
     def test_export(self):
         appointment = create_appointment(visit=create_visit())
-        appointment.comment = 'comm_ent'
+        appointment.comment = "comm_ent"
         appointment.save()
 
         exporter = CsvVisitExporter(self.visit_export_data)
         exporter.execute()
 
-        with open(self.visit_export_data.get_absolute_file_path(), 'r') as file:
+        with open(self.visit_export_data.get_absolute_file_path(), "r", encoding="utf-8") as file:
             data = file.read()
             self.assertTrue(appointment.comment in data)
 
     def test_export_subject_number(self):
-        create_appointment(visit=create_visit(subject=create_study_subject(nd_number='ND0123')))
+        create_appointment(visit=create_visit(subject=create_study_subject(nd_number="ND0123")))
 
         exporter = CsvVisitExporter(self.visit_export_data)
         exporter.execute()
 
-        with open(self.visit_export_data.get_absolute_file_path(), 'r') as file:
+        with open(self.visit_export_data.get_absolute_file_path(), "r", encoding="utf-8") as file:
             data = file.read()
-            self.assertTrue('ND0123' in data)
+            self.assertTrue("ND0123" in data)
 
     def test_export_general_appointment(self):
         appointment = create_appointment()
         appointment.visit = None
-        appointment.comment = 'comm_ent'
+        appointment.comment = "comm_ent"
         appointment.save()
 
         exporter = CsvVisitExporter(self.visit_export_data)
         exporter.execute()
 
-        with open(self.visit_export_data.get_absolute_file_path(), 'r') as file:
+        with open(self.visit_export_data.get_absolute_file_path(), "r", encoding="utf-8") as file:
             data = file.read()
             self.assertTrue(appointment.comment in data)
 
@@ -77,9 +77,9 @@ class TestVisitExporter(TestCase):
         exporter = CsvVisitExporter(self.visit_export_data)
         exporter.execute()
 
-        with open(self.visit_export_data.get_absolute_file_path(), 'r') as file:
+        with open(self.visit_export_data.get_absolute_file_path(), "r", encoding="utf-8") as file:
             data = file.read()
-            self.assertTrue('visit_number' in data)
+            self.assertTrue("visit_number" in data)
             self.assertTrue('"' + str(appointment.visit.visit_number) + '"' in data)
 
     def test_export_when(self):
@@ -90,6 +90,6 @@ class TestVisitExporter(TestCase):
         exporter = CsvVisitExporter(self.visit_export_data)
         exporter.execute()
 
-        with open(self.visit_export_data.get_absolute_file_path(), 'r') as file:
+        with open(self.visit_export_data.get_absolute_file_path(), "r", encoding="utf-8") as file:
             data = file.read()
             self.assertTrue(appointment.datetime_when.strftime(self.visit_export_data.date_format) in data)