From 4483cf6166cacf7d54335be4188af9810e6d12b4 Mon Sep 17 00:00:00 2001
From: Carlos Vega <carlos.vega@uni.lu>
Date: Mon, 9 Nov 2020 17:06:29 +0100
Subject: [PATCH] added dynamic table that loads via AJAX. This should support
thousands of rows without issue
---
requirements.txt | 1 +
smash/web/api_urls.py | 4 +-
smash/web/api_views/provenance.py | 86 ++++++++++++++++++++++++
smash/web/templates/export/index.html | 58 +++++++++++-----
smash/web/templates/provenance/list.html | 83 +++++++++++++++++------
smash/web/utils.py | 7 ++
6 files changed, 202 insertions(+), 37 deletions(-)
create mode 100644 smash/web/api_views/provenance.py
diff --git a/requirements.txt b/requirements.txt
index 343e3de8..36a5df54 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -33,6 +33,7 @@ mockito==1.2.2
nexmo==2.3.0
numpy==1.16.1
pandas==1.1.3
+django-datatables-view==1.19.1
phonenumberslite==8.9.14
Pillow==3.4.2
psycopg2==2.8.6
diff --git a/smash/web/api_urls.py b/smash/web/api_urls.py
index 22f3d57d..7dae498a 100644
--- a/smash/web/api_urls.py
+++ b/smash/web/api_urls.py
@@ -16,7 +16,7 @@ Including another URLconf
from django.conf.urls import url
from web.api_views import worker, location, subject, appointment_type, appointment, configuration, daily_planning, \
- redcap, flying_team, visit, voucher, voucher_type
+ redcap, flying_team, visit, voucher, voucher_type, provenance
urlpatterns = [
# appointments
@@ -35,6 +35,8 @@ urlpatterns = [
# subjects data
url(r'^cities$', subject.cities, name='web.api.cities'),
url(r'^referrals$', subject.referrals, name='web.api.referrals'),
+ url(r'^export_log$', provenance.ExportLog.as_view(), name='web.api.export_log'),
+ url(r'^provenances$', provenance.CompleteLog.as_view(), name='web.api.provenances'),
url(r'^subjects/(?P<type>[A-z]+)$', subject.subjects, name='web.api.subjects'),
url(r'^subjects:columns/(?P<subject_list_type>[A-z]+)$', subject.get_subject_columns,
name='web.api.subjects.columns'),
diff --git a/smash/web/api_views/provenance.py b/smash/web/api_views/provenance.py
new file mode 100644
index 00000000..6fe1407d
--- /dev/null
+++ b/smash/web/api_views/provenance.py
@@ -0,0 +1,86 @@
+from ..models import Provenance, Worker
+from django_datatables_view.base_datatable_view import BaseDatatableView
+
+class ExportLog(BaseDatatableView):
+ model = Provenance
+ columns = [
+ 'modification_date',
+ 'modification_author',
+ 'modification_description',
+ 'request_path',
+ 'request_ip_addr'
+ ]
+
+ # define column names that will be used in sorting
+ # order is important and should be same as order of columns
+ # displayed by datatables. For non sortable columns use empty
+ # value like ''
+ order_columns = [
+ 'modification_date',
+ 'modification_author',
+ 'modification_description',
+ 'request_path',
+ 'request_ip_addr'
+ ]
+
+ # set max limit of records returned, this is used to protect our site if someone tries to attack our site
+ # and make it return huge amount of data
+ max_display_length = 10
+
+ def filter_queryset(self, qs):
+ # simple example:
+ search = self.request.GET.get('search[value]', None)
+ if search != '' and search is not None:
+ inner_qs = Worker.objects.filter(first_name__icontains=search) | Worker.objects.filter(last_name__icontains=search)
+ qs = qs.filter(modification_author__in=inner_qs) | qs.filter(modification_description__icontains=search) | qs.filter(request_path__icontains=search) | qs.filter(request_ip_addr__icontains=search)
+
+ qs = qs & qs.filter(modified_table__isnull=True,
+ previous_value__isnull=True,
+ new_value__isnull=True,
+ modified_field='',
+ request_path__startswith='/export/')
+
+ return qs
+
+class CompleteLog(BaseDatatableView):
+ model = Provenance
+ columns = ['modified_table',
+ 'modified_table_id',
+ 'modification_date',
+ 'modification_author',
+ 'modified_field',
+ 'previous_value',
+ 'new_value',
+ 'modification_description',
+ 'request_path',
+ 'request_ip_addr'
+ ]
+
+ # define column names that will be used in sorting
+ # order is important and should be same as order of columns
+ # displayed by datatables. For non sortable columns use empty
+ # value like ''
+ order_columns = ['modified_table',
+ 'modified_table_id',
+ 'modification_date',
+ 'modification_author',
+ 'modified_field',
+ 'previous_value',
+ 'new_value',
+ 'modification_description',
+ 'request_path',
+ 'request_ip_addr'
+ ]
+
+ # set max limit of records returned, this is used to protect our site if someone tries to attack our site
+ # and make it return huge amount of data
+ max_display_length = 10
+
+ def filter_queryset(self, qs):
+ # simple example:
+ search = self.request.GET.get('search[value]', None)
+ if search != '' and search is not None:
+ inner_qs = Worker.objects.filter(first_name__icontains=search) | Worker.objects.filter(last_name__icontains=search)
+ qs = qs.filter(modification_author__in=inner_qs) | qs.filter(modification_description__icontains=search) | qs.filter(request_path__icontains=search) | qs.filter(request_ip_addr__icontains=search)
+
+ return qs
\ No newline at end of file
diff --git a/smash/web/templates/export/index.html b/smash/web/templates/export/index.html
index 112fe330..4e99d70b 100644
--- a/smash/web/templates/export/index.html
+++ b/smash/web/templates/export/index.html
@@ -105,20 +105,15 @@
<th>Author</th>
<th>Description</th>
<th>Request Path</th>
+ <th>Request IP</th>
</tr>
</thead>
<tbody>
- {% for provenance in provenances %}
- <tr>
- <td data-sort="{{ provenance.modification_date | timestamp }}">{{ provenance.modification_date }}</td>
- <td>{{ provenance.modification_author }}</td>
- <td>{{ provenance.modification_description }}</td>
- <td>{{ provenance.request_path }}</td>
- </tr>
- {% endfor %}
+
</tbody>
</table>
</div>
+
</div>
<div class="box-body">
@@ -139,6 +134,9 @@
$(e.target).children('i.fa').toggleClass('fa-expand');
$(e.target).children('i.fa').toggleClass('fa-compress');
});
+
+
+
};
function addFields(e, cls){
var fields = $(`.${cls} > li > input:checked`).map(function(i,e){
@@ -161,14 +159,42 @@
}
$(function () {
- $('#table').DataTable({
- "paging": true,
- "lengthChange": false,
- "searching": true,
- "ordering": true,
- "order": [[ 0, "desc" ]],
- "info": true,
- "autoWidth": false
+ console.log("{% url 'web.api.provenances' %}")
+ var oTable = $('#table').DataTable({
+ processing: true,
+ serverSide: true,
+ ordering: true,
+ autoWidth: false,
+ lengthChange: false,
+ columns: [
+ {
+ data: 'modification_date',
+ orderable: true,
+ searchable: true
+ },
+ {
+ data: 'modification_author',
+ orderable: true,
+ searchable: true,
+ },
+ {
+ data: 'modification_description',
+ orderable: true,
+ searchable: true,
+ },
+ {
+ data: 'request_path',
+ orderable: true,
+ searchable: true,
+ },
+ {
+ data: 'request_ip_addr',
+ orderable: true,
+ searchable: true,
+ }
+ ],
+ order: [[ 0, "desc" ]],
+ ajax: "{% url 'web.api.export_log' %}"
});
});
</script>
diff --git a/smash/web/templates/provenance/list.html b/smash/web/templates/provenance/list.html
index 44910e46..34051837 100644
--- a/smash/web/templates/provenance/list.html
+++ b/smash/web/templates/provenance/list.html
@@ -31,22 +31,10 @@
<th>New Value</th>
<th>Description</th>
<th>Request Path</th>
+ <th>Request IP</th>
</tr>
</thead>
<tbody>
- {% for provenance in provenances %}
- <tr>
- <td>{{ provenance.modified_table }}</td>
- <td>{{ provenance.modified_table_id }}</td>
- <td data-sort="{{ provenance.modification_date | timestamp }}">{{ provenance.modification_date }}</td>
- <td>{{ provenance.modification_author }}</td>
- <td>{{ provenance.modified_field }}</td>
- <td>{{ provenance.previous_value }}</td>
- <td>{{ provenance.new_value }}</td>
- <td>{{ provenance.modification_description }}</td>
- <td>{{ provenance.request_path }}</td>
- </tr>
- {% endfor %}
</tbody>
</table>
</div>
@@ -61,13 +49,68 @@
<script>
$(function () {
$('#table').DataTable({
- "paging": true,
- "lengthChange": false,
- "searching": true,
- "ordering": true,
- "order": [[ 2, "desc" ]],
- "info": true,
- "autoWidth": false
+ paging: true,
+ lengthChange: false,
+ searching: true,
+ processing: true,
+ serverSide: true,
+ ordering: true,
+ order: [[ 2, "desc" ]],
+ info: true,
+ autoWidth: false,
+ columns: [
+ {
+ data: 'modified_table',
+ orderable: true,
+ searchable: true
+ },
+ {
+ data: 'modified_table_id',
+ orderable: true,
+ searchable: true
+ },
+ {
+ data: 'modification_date',
+ orderable: true,
+ searchable: true
+ },
+ {
+ data: 'modification_author',
+ orderable: true,
+ searchable: true,
+ },
+ {
+ data: 'modified_field',
+ orderable: true,
+ searchable: true,
+ },
+ {
+ data: 'previous_value',
+ orderable: true,
+ searchable: true,
+ },
+ {
+ data: 'new_value',
+ orderable: true,
+ searchable: true,
+ },
+ {
+ data: 'modification_description',
+ orderable: true,
+ searchable: true,
+ },
+ {
+ data: 'request_path',
+ orderable: true,
+ searchable: true,
+ },
+ {
+ data: 'request_ip_addr',
+ orderable: true,
+ searchable: true,
+ }
+ ],
+ ajax: "{% url 'web.api.provenances' %}"
});
});
</script>
diff --git a/smash/web/utils.py b/smash/web/utils.py
index dd61c944..54666468 100644
--- a/smash/web/utils.py
+++ b/smash/web/utils.py
@@ -5,6 +5,13 @@ from datetime import timedelta
from web.algorithm import VerhoeffAlgorithm, LuhnAlgorithm
+def get_client_ip(request):
+ x_forwarded_for = request.META.get('HTTP_X_FORWARDED_FOR')
+ if x_forwarded_for:
+ ip = x_forwarded_for.split(',')[0]
+ else:
+ ip = request.META.get('REMOTE_ADDR')
+ return ip
def is_valid_social_security_number(number):
if number is not None and number != '':
--
GitLab