Merge branch 'development' into IVS-556-task-pasta-refactoring

Author: Ghesselink

.github/workflows/ci_cd.yml

@@ -10,9 +10,6 @@ on:
       - development
     paths-ignore:
       - 'README.md'
-  pull_request:
-    branches:
-      - development
   workflow_dispatch:
 
 jobs:

Makefile

@@ -75,6 +75,12 @@ fetch-modules:
 	git submodule foreach git reset --hard
 	git submodule update --remote --recursive
 
+# runs end-to-end tests against a local instance of the Validation Service DB
+e2e-test: start-infra
+	cd e2e && npm install && npm run install-playwright && npm run test
+
+e2e-test-report: start-infra
+	cd e2e && npm install && npm run install-playwright && npm run test:html && npm run test:report
 
 BRANCH   ?= main                 
 SUBTREES := \
@@ -97,4 +103,4 @@ checkout:
 
 	@echo "==> signatures/store (always on main)"
 	@( cd backend/apps/ifc_validation/checks/signatures/store && \
-	   git checkout -q main && git pull )
+	   git checkout -q main && git pull )

backend/apps/ifc_validation/admin.py

@@ -26,6 +26,7 @@
 from apps.ifc_validation_models.models import set_user_context
 
 from .tasks import ifc_file_validation_task
+from .filters import ProducedByAdvancedFilter, ModelProducedByAdvancedFilter, CreatedByAdvancedFilter
 
 from core import utils
 from core.filters import AdvancedDateFilter
@@ -65,7 +66,15 @@ class ValidationRequestAdmin(BaseAdmin, NonAdminAddable):
     readonly_fields = ["id", "public_id", "deleted", "file_name", "file", "file_size_text", "duration_text", "started", "completed", "channel", "created", "created_by", "updated", "updated_by"] 
     date_hierarchy = "created"
 
-    list_filter = ["status", "deleted", "model__produced_by", "channel", "created_by", "created_by__useradditionalinfo__is_vendor", "created_by__useradditionalinfo__is_vendor_self_declared", ('created', AdvancedDateFilter)]
+    list_filter = [
+        "status", 
+        "deleted", 
+        ModelProducedByAdvancedFilter, 
+        "channel", 
+        CreatedByAdvancedFilter, 
+        "created_by__useradditionalinfo__is_vendor", 
+        "created_by__useradditionalinfo__is_vendor_self_declared", 
+        ('created', AdvancedDateFilter)]
     search_fields = ('file_name', 'status', 'model__produced_by__name', 'created_by__username', 'updated_by__username')
 
     actions = ["soft_delete_action", "soft_restore_action", "mark_as_failed_action", "restart_processing_action", "hard_delete_action"]
@@ -363,7 +372,12 @@ class ModelAdmin(BaseAdmin, NonAdminAddable):
     date_hierarchy = "created"
 
     search_fields = ('file_name', 'schema', 'mvd', 'produced_by__name', 'produced_by__version')
-    list_filter = ['schema', 'produced_by', ('date', AdvancedDateFilter), ('created', AdvancedDateFilter)]
+    list_filter = [
+        'schema', 
+        ProducedByAdvancedFilter,
+        ('date', AdvancedDateFilter), 
+        ('created', AdvancedDateFilter)
+    ]
 
     @admin.display(description="File Size", ordering='size')
     def size_text(self, obj):

backend/apps/ifc_validation/checks/ifc_gherkin_rules

@@ -0,0 +1,73 @@
+from django.db.models import Q
+from django.utils.translation import gettext_lazy as _
+
+from core.filters import AdvancedInputFilter
+
+
+class ModelProducedByAdvancedFilter(AdvancedInputFilter):
+
+    parameter_name = 'model__produced_by__contains'
+    title = _('Produced By')
+
+    def queryset(self, request, queryset):
+
+        term = self.value()
+
+        if term is None:
+            return
+
+        any_name = Q()
+        for bit in term.split():
+            any_name &= (
+                Q(model__produced_by__name__icontains=bit) |
+                Q(model__produced_by__version__icontains=bit) |
+                Q(model__produced_by__company__name__icontains=bit)
+            )
+
+        return queryset.filter(any_name)
+    
+
+class ProducedByAdvancedFilter(AdvancedInputFilter):
+
+    parameter_name = 'produced_by__contains'
+    title = _('Produced By')
+
+    def queryset(self, request, queryset):
+
+        term = self.value()
+
+        if term is None:
+            return
+
+        any_name = Q()
+        for bit in term.split():
+            any_name &= (
+                Q(produced_by__name__icontains=bit) |
+                Q(produced_by__version__icontains=bit) |
+                Q(produced_by__company__name__icontains=bit)
+            )
+
+        return queryset.filter(any_name)
+    
+
+class CreatedByAdvancedFilter(AdvancedInputFilter):
+
+    parameter_name = 'created_by__contains'
+    title = _('Created By')
+
+    def queryset(self, request, queryset):
+
+        term = self.value()
+
+        if term is None:
+            return
+
+        any = Q()
+        for bit in term.split():
+            any &= (
+                Q(created_by__username=bit) |
+                Q(created_by__last_name__icontains=bit) |
+                Q(created_by__first_name__icontains=bit)
+            )
+
+        return queryset.filter(any)

backend/apps/ifc_validation/filters.py

@@ -55,11 +55,12 @@ def send_acknowledgement_admin_email_task(id, file_name):
         'USER_FULL_NAME': user.get_full_name(),
         'USER_EMAIL': user.email,
         'PUBLIC_URL': PUBLIC_URL,
-        'ENVIRONMENT': ENVIRONMENT
+        'ENVIRONMENT': ENVIRONMENT,
+        'CHANNEL': request.channel
     }
     to = ADMIN_EMAIL
     body_html = render_to_string("validation_ack_admin_email.html", merge_data)
-    body_text = f"User uploaded {{merge_data.NUMBER_OF_FILES}} file(s) in {{merge_data.ENVIRONMENT}}."
+    body_text = f"User submitted {{merge_data.NUMBER_OF_FILES}} file(s) in {{merge_data.ENVIRONMENT}}."
     subject = get_title_from_html(body_html)
 
     # queue for sending

backend/apps/ifc_validation/tasks/email_tasks.py

@@ -1,10 +1,10 @@
 <html>
     <head>
-        <title>Validation Service - User uploaded {{NUMBER_OF_FILES}} file(s)</title>
+        <title>Validation Service - User submitted {{NUMBER_OF_FILES}} file(s)</title>
     </head>
     <body>
         <div>
-            {{NUMBER_OF_FILES}} file(s) were uploaded by {{USER_FULL_NAME}} ({{USER_EMAIL}}): {{FILE_NAMES}} in {{ENVIRONMENT}}.
+            {{NUMBER_OF_FILES}} file(s) were submitted by {{USER_FULL_NAME}} ({{USER_EMAIL}}): {{FILE_NAMES}} in {{ENVIRONMENT}} via {{CHANNEL}}.
         </div>
     </body>
 </html>

backend/apps/ifc_validation/templates/validation_ack_admin_email.html

@@ -4,7 +4,7 @@
     </head>
     <body>
         <div>
-            {{NUMBER_OF_FILES}} file(s) uploaded by {{USER_FULL_NAME}} ({{USER_EMAIL}}) failed to process: {{FILE_NAMES}} in {{ENVIRONMENT}}.
+            {{NUMBER_OF_FILES}} file(s) submitted by {{USER_FULL_NAME}} ({{USER_EMAIL}}) failed to process: {{FILE_NAMES}} in {{ENVIRONMENT}}.
         </div>
     </body>
 </html>

backend/apps/ifc_validation/templates/validation_failed_admin_email.html

@@ -1,4 +1,4 @@
-from django.urls import path
+from django.urls import path, re_path
 
 from .views import ValidationRequestListAPIView, ValidationRequestDetailAPIView
 from .views import ValidationTaskListAPIView, ValidationTaskDetailAPIView
@@ -11,14 +11,15 @@
 urlpatterns = [
 
     # REST API
-    path('validationrequest/',          ValidationRequestListAPIView.as_view()),
-    path('validationrequest/<str:id>/', ValidationRequestDetailAPIView.as_view()),
-    path('validationtask/',             ValidationTaskListAPIView.as_view()),
-    path('validationtask/<str:id>/',    ValidationTaskDetailAPIView.as_view()),
-    path('validationoutcome/',          ValidationOutcomeListAPIView.as_view()),
-    path('validationoutcome/<str:id>/', ValidationOutcomeDetailAPIView.as_view()),
-    path('model/',                      ModelListAPIView.as_view()),
-    path('model/<str:id>/',             ModelDetailAPIView.as_view()),
+    # using re_path to make trailing slashes optional
+    re_path(r'validationrequest/?$',                ValidationRequestListAPIView.as_view()),
+    re_path(r'validationrequest/(?P<id>[\w-]+)/?$', ValidationRequestDetailAPIView.as_view()),
+    re_path(r'validationtask/?$',                   ValidationTaskListAPIView.as_view()),
+    re_path(r'validationtask/(?P<id>[\w-]+)/?$',    ValidationTaskDetailAPIView.as_view()),
+    re_path(r'validationoutcome/?$',                ValidationOutcomeListAPIView.as_view()),
+    re_path(r'validationoutcome/(?P<id>[\w-]+)/?$', ValidationOutcomeDetailAPIView.as_view()),
+    re_path(r'model/?$',                            ModelListAPIView.as_view()),
+    re_path(r'model/(?P<id>[\w-]+)/?$',             ModelDetailAPIView.as_view()),
 
     # Django Admin charts
     path("chart/filter-options/", charts.get_filter_options),

backend/apps/ifc_validation/urls.py

@@ -4,15 +4,14 @@
 
 from django.db import transaction
 from core.utils import get_client_ip_address
-from core.settings import MAX_FILES_PER_UPLOAD
+from core.settings import MAX_FILES_PER_UPLOAD, MAX_FILE_SIZE_IN_MB
 
 from rest_framework import status
 from rest_framework.parsers import FormParser, MultiPartParser
 from rest_framework.response import Response
 from rest_framework.views import APIView
 from rest_framework.exceptions import APIException
 from rest_framework.permissions import IsAuthenticated
-from rest_framework.authentication import SessionAuthentication, BasicAuthentication, TokenAuthentication
 from rest_framework.throttling import UserRateThrottle
 from rest_framework.throttling import ScopedRateThrottle
 from rest_framework.decorators import throttle_classes
@@ -33,7 +32,6 @@
 class ValidationRequestDetailAPIView(APIView):
 
     queryset = ValidationRequest.objects.all()
-    authentication_classes = [SessionAuthentication, TokenAuthentication, BasicAuthentication]
     permission_classes = [IsAuthenticated]
     parser_classes = (MultiPartParser, FormParser)
     serializer_class = ValidationRequestSerializer
@@ -78,7 +76,6 @@ def delete(self, request, id, *args, **kwargs):
 class ValidationRequestListAPIView(APIView):
 
     queryset = ValidationRequest.objects.all()
-    authentication_classes = [SessionAuthentication, TokenAuthentication, BasicAuthentication]
     permission_classes = [IsAuthenticated]
     parser_classes = (MultiPartParser, FormParser)
     serializer_class = ValidationRequestSerializer
@@ -90,7 +87,6 @@ def get_throttles(self):
         """
         Applies scoped throttling only for POST requests (aka submitting a new Validation Request).
         """    
-        logger.info('*** ' + self.request.method)
         return [ScopedRateThrottle()] if self.request.method == 'POST' else [UserRateThrottle()]
 
     @extend_schema(operation_id='validationrequest_list')
@@ -103,6 +99,10 @@ def get(self, request, *args, **kwargs):
         logger.info('API request - User IP: %s Request Method: %s Request URL: %s Content-Length: %s' % (get_client_ip_address(request), request.method, request.path, request.META.get('CONTENT_LENGTH')))
 
         user_requests = ValidationRequest.objects.filter(created_by__id=request.user.id, deleted=False)
+        public_id = self.request.query_params.get('public_id', None)
+        if public_id:
+            user_requests = user_requests.filter(id=ValidationRequest.to_private_id(public_id))
+
         serializer = self.serializer_class(user_requests, many=True)
         return Response(serializer.data, status=status.HTTP_200_OK)
 
@@ -132,14 +132,29 @@ def post(self, request, *args, **kwargs):
                         if file_i is not None: files += file_i
                     logger.info(f"Received {len(files)} file(s) - files: {files}")
 
+                    # only accept one file (for now)
+                    if len(files) != 1:
+                        data = {'message': f"Only one file can be uploaded at a time."}
+                        return Response(data, status=status.HTTP_400_BAD_REQUEST)
+
                     # retrieve file size and save
                     uploaded_file = serializer.validated_data
                     logger.info(f'uploaded_file = {uploaded_file}')
                     f = uploaded_file['file']
                     f.seek(0, 2)
                     file_length = f.tell()
                     file_name = uploaded_file['file_name']
-                    logger.info(f"file_length for uploaded file {file_name} = {file_length}")
+                    logger.info(f"file_length for uploaded file {file_name} = {file_length} ({file_length / (1024*1024)} MB)")
+
+                    # check if file name ends with .ifc
+                    if not file_name.lower().endswith('.ifc'):
+                        data = {'file_name': "File name must end with '.ifc'."}
+                        return Response(data, status=status.HTTP_400_BAD_REQUEST)
+
+                    # apply file size limit
+                    if file_length > MAX_FILE_SIZE_IN_MB * 1024 * 1024:
+                        data = {'message': f"File size exceeds allowed file size limit ({MAX_FILE_SIZE_IN_MB} MB)."}
+                        return Response(data, status=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE)
 
                     # can't use this, file hasn't been saved yet
                     #file = os.path.join(MEDIA_ROOT, uploaded_file['file_name'])                   
@@ -167,7 +182,6 @@ def submit_task(instance):
 class ValidationTaskDetailAPIView(APIView):
 
     queryset = ValidationTask.objects.all()
-    authentication_classes = [SessionAuthentication, BasicAuthentication, TokenAuthentication]
     permission_classes = [IsAuthenticated]
     serializer_class = ValidationTaskSerializer
     throttle_classes = [UserRateThrottle]
@@ -193,7 +207,6 @@ def get(self, request, id, *args, **kwargs):
 class ValidationTaskListAPIView(APIView):
 
     queryset = ValidationTask.objects.all()
-    authentication_classes = [SessionAuthentication, BasicAuthentication, TokenAuthentication]
     permission_classes = [IsAuthenticated]
     serializer_class = ValidationTaskSerializer
     throttle_classes = [UserRateThrottle]
@@ -224,7 +237,6 @@ def get(self, request, *args, **kwargs):
 class ValidationOutcomeDetailAPIView(APIView):
 
     queryset = ValidationOutcome.objects.all()
-    authentication_classes = [SessionAuthentication, BasicAuthentication, TokenAuthentication]
     permission_classes = [IsAuthenticated]
     serializer_class = ValidationOutcomeSerializer
     throttle_classes = [UserRateThrottle]
@@ -250,7 +262,6 @@ def get(self, request, id, *args, **kwargs):
 class ValidationOutcomeListAPIView(APIView):
 
     queryset = ValidationOutcome.objects.all()
-    authentication_classes = [SessionAuthentication, BasicAuthentication, TokenAuthentication]
     permission_classes = [IsAuthenticated]
     serializer_class = ValidationOutcomeSerializer
     throttle_classes = [UserRateThrottle]
@@ -285,7 +296,6 @@ def get(self, request, *args, **kwargs):
 class ModelDetailAPIView(APIView):
 
     queryset = Model.objects.all()
-    authentication_classes = [SessionAuthentication, BasicAuthentication, TokenAuthentication]
     permission_classes = [IsAuthenticated]
     serializer_class = ModelSerializer
     throttle_classes = [UserRateThrottle]
@@ -311,7 +321,6 @@ def get(self, request, id, *args, **kwargs):
 class ModelListAPIView(APIView):
 
     queryset = Model.objects.all()
-    authentication_classes = [SessionAuthentication, BasicAuthentication, TokenAuthentication]
     permission_classes = [IsAuthenticated]
     serializer_class = ModelSerializer
     throttle_classes = [UserRateThrottle]