refactor data access part 1 models validators [Please donot merge]

physionet-django/console/forms.py

@@ -21,6 +21,7 @@
     Contact,
     CopyeditLog,
     DataAccess,
+    DataSource,
     DUA,
     EditLog,
     License,
@@ -689,6 +690,26 @@ def save(self):
         return data_access
 
 
+class DataSourceForm(forms.ModelForm):
+    class Meta:
+        model = DataSource
+        fields = ('data_location', 'access_mechanism', 'files_available', 'email', 'uri' )
+
+    def __init__(self, project, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.project = project
+
+        if not settings.ENABLE_CLOUD_RESEARCH_ENVIRONMENTS:
+            self.fields['access_mechanism'].choices = [
+                choice for choice in self.fields['access_mechanism'].choices if choice[0] != 'research-environment']
+
+    def save(self):
+        data_source = super(DataSourceForm, self).save(commit=False)
+        data_source.project = self.project
+        data_source.save()
+        return data_source
+
+
 class PublishedProjectContactForm(forms.ModelForm):
     class Meta:
         model = Contact

physionet-django/console/templates/console/manage_published_project.html

@@ -359,6 +359,46 @@ <h5 class="card-title mt-0 mb-1">Storage location</h5>
         </table>
         {% endif %}
       </li>
+      <li class="list-group-item">
+        <h5 class="card-title mt-0 mb-1">Data Source</h5>
+        <p>Add and remove Data Source options.</p>
+{#        <div class="alert alert-danger">#}
+{#          <p class='m-0'>Note: The remove button will remove the option for requesting cloud access that appears in the files section of a project. It will not (1) delete/deactivate the bucket or (2) remove access for users who are already using the bucket.</p>#}
+{#        </div>#}
+        <form action="" method="post">
+          {% csrf_token %}
+          {% include "project/content_inline_form_snippet.html" with form=data_source_form %}
+          <button class="btn btn-primary" type="submit">Submit</button>
+        </form>
+        {% if data_sources %}
+        <table class="table table-bordered">
+            <tr>
+              <th>Location</th>
+              <th>Access Mechanism</th>
+              <th>Files Available</th>
+              <th>Email</th>
+              <th>Uri</th>
+              <th>Remove</th>
+            </tr>
+          </thead>
+          <tbody>
+            {% for item in data_sources %}
+            <tr>
+              <td>{{item.data_location}}</td>
+              <td>{{item.access_mechanism}}</td>
+              <td>{{item.files_available}}</td>
+              <td>{{item.email}}</td>
+              <td>{{item.uri}}</td>
+              <form action="" method="post">
+                {% csrf_token %}
+                <td><button class='btn btn-danger' name='data_source_removal' value='{{item.id}}'>Remove</button></td>
+              </form>
+            </tr>
+            {% endfor %}
+          </tbody>
+        </table>
+        {% endif %}
+      </li>
       <li class="list-group-item">
         <h5 class="card-title mt-3 mb-1">Google Cloud</h5>
         {% if not has_credentials %}
@@ -388,6 +428,7 @@ <h5 class="card-title mt-3 mb-1">Google Cloud</h5>
     </ul>
 
   </div>
+
 </div>
 
 {% endblock %}

physionet-django/console/views.py

@@ -43,6 +43,7 @@
     ActiveProject,
     ArchivedProject,
     DataAccess,
+    DataSource,
     DUA,
     DataAccessRequest,
     DUASignature,
@@ -829,6 +830,7 @@ def manage_published_project(request, project_slug, version):
     deprecate_form = None if project.deprecated_files else forms.DeprecateFilesForm()
     has_credentials = bool(settings.GOOGLE_APPLICATION_CREDENTIALS)
     data_access_form = forms.DataAccessForm(project=project)
+    data_source_form = forms.DataSourceForm(project=project)
     contact_form = forms.PublishedProjectContactForm(project=project,
                                                      instance=project.contact)
     legacy_author_form = forms.CreateLegacyAuthorForm(project=project)
@@ -895,6 +897,18 @@ def manage_published_project(request, project_slug, version):
             if data_access_form.is_valid():
                 data_access_form.save()
                 messages.success(request, "Stored method to access the files")
+        elif 'data_location' in request.POST:
+            data_source_form = forms.DataSourceForm(project=project, data=request.POST)
+            if data_source_form.is_valid():
+                data_source_form.save()
+                messages.success(request, "Stored method to access the files")
+        elif 'data_source_removal' in request.POST and request.POST['data_source_removal'].isdigit():
+            try:
+                data_source = DataSource.objects.get(project=project, id=request.POST['data_source_removal'])
+                data_source.delete()
+                # Deletes the object if it exists for that specific project.
+            except DataSource.DoesNotExist:
+                pass
         elif 'data_access_removal' in request.POST and request.POST['data_access_removal'].isdigit():
             try:
                 data_access = DataAccess.objects.get(project=project, id=request.POST['data_access_removal'])
@@ -921,6 +935,7 @@ def manage_published_project(request, project_slug, version):
                 legacy_author_form = forms.CreateLegacyAuthorForm(project=project)
 
     data_access = DataAccess.objects.filter(project=project)
+    data_sources = DataSource.objects.filter(project=project)
     authors, author_emails, storage_info, edit_logs, copyedit_logs, latest_version = project.info_card()
 
     tasks = list(get_associated_tasks(project))
@@ -946,7 +961,9 @@ def manage_published_project(request, project_slug, version):
             'deprecate_form': deprecate_form,
             'has_credentials': has_credentials,
             'data_access_form': data_access_form,
+            'data_source_form': data_source_form,
             'data_access': data_access,
+            'data_sources': data_sources,
             'rw_tasks': rw_tasks,
             'ro_tasks': ro_tasks,
             'anonymous_url': anonymous_url,

physionet-django/project/modelcomponents/access.py

@@ -4,6 +4,7 @@
 from django.contrib.auth.hashers import check_password, make_password
 from django.contrib.contenttypes.fields import GenericForeignKey
 from django.contrib.contenttypes.models import ContentType
+from django.core.exceptions import ValidationError
 from django.db import models
 from django.utils import timezone
 from django.utils.crypto import get_random_string
@@ -167,6 +168,87 @@ class Meta:
         default_permissions = ()
 
 
+class DataSource(models.Model):
+    """
+    Controls all access to project data.
+    """
+    class DataLocation(models.TextChoices):
+        DIRECT = 'Direct'
+        GOOGLE_BIGQUERY = 'Google BigQuery'
+        GOOGLE_CLOUD_STORAGE = 'Google Cloud Storage'
+        AWS_OPEN_DATA = 'AWS Open Data'
+        AWS_S3 = 'AWS S3'
+
+    class AccessMechanism(models.TextChoices):
+        GOOGLE_GROUP_EMAIL = 'Google Group Email'
+        S3 = 'S3'
+        RESEARCH_ENVIRONMENT = 'Research Environment'
+
+    project = models.ForeignKey('project.PublishedProject',
+                                related_name='data_sources', db_index=True, on_delete=models.CASCADE)
+    files_available = models.BooleanField(default=False)
+    data_location = models.CharField(max_length=20, choices=DataLocation.choices)
+    access_mechanism = models.CharField(max_length=20, choices=AccessMechanism.choices, null=True, blank=True)
+    email = models.EmailField(max_length=320, null=True, blank=True)
+    uri = models.CharField(max_length=320, null=True, blank=True)
+
+    class Meta:
+        default_permissions = ()
+        unique_together = ('project', 'data_location')
+
+    def clean(self):
+        super().clean()
+
+        # all the fields in the list are expected to be present for the given data_location
+        required_fields_data_location = {
+            self.DataLocation.GOOGLE_BIGQUERY: ["email"],
+            self.DataLocation.GOOGLE_CLOUD_STORAGE: ["uri"],
+            self.DataLocation.AWS_OPEN_DATA: ["uri"],
+            self.DataLocation.AWS_S3: ["uri"],
+            # self.DataLocation.DIRECT: []
+        }
+        # one of the access_mechanism in the list is expected to be present for the given data_location
+        required_access_mechanism_data_location = {
+            self.DataLocation.GOOGLE_BIGQUERY: [self.AccessMechanism.GOOGLE_GROUP_EMAIL],
+            self.DataLocation.GOOGLE_CLOUD_STORAGE: [self.AccessMechanism.GOOGLE_GROUP_EMAIL,
+                                                     self.AccessMechanism.RESEARCH_ENVIRONMENT],
+            self.DataLocation.AWS_OPEN_DATA: [self.AccessMechanism.S3],
+            self.DataLocation.AWS_S3: [self.AccessMechanism.S3],
+            # self.DataLocation.DIRECT: []
+        }
+        # None of the access_mechanism in the list are expected be present for the given data_location
+        forbidden_access_mechanism_data_location = {
+            self.DataLocation.DIRECT: [self.AccessMechanism.GOOGLE_GROUP_EMAIL, self.AccessMechanism.S3,
+                                       self.AccessMechanism.RESEARCH_ENVIRONMENT]
+        }
+        # None the fields in the list are expected to not be present for the given data_location
+        forbidden_fields_data_location = {
+            self.DataLocation.DIRECT: ["uri", "email"]
+        }
+
+        if self.data_location in required_access_mechanism_data_location:
+            if self.access_mechanism not in required_access_mechanism_data_location.get(self.data_location, []):
+                raise ValidationError(
+                    f'{self.data_location} data sources must use one of the following access mechanisms: '
+                    f'{", ".join(required_access_mechanism_data_location.get(self.data_location, []))}.')
+
+        if self.data_location in forbidden_access_mechanism_data_location:
+            if self.access_mechanism in forbidden_access_mechanism_data_location.get(self.data_location, []):
+                raise ValidationError(
+                    f'{self.data_location} data sources must not use the following access mechanisms: '
+                    f'{", ".join(forbidden_access_mechanism_data_location.get(self.data_location, []))}.')
+
+        if self.data_location in required_fields_data_location:
+            for required_field in required_fields_data_location.get(self.data_location, []):
+                if not getattr(self, required_field):
+                    raise ValidationError(f'{self.data_location} data sources must have a {required_field}.')
+
+        if self.data_location in forbidden_fields_data_location:
+            for forbidden_field in forbidden_fields_data_location.get(self.data_location, []):
+                if getattr(self, forbidden_field):
+                    raise ValidationError(f'{self.data_location} sources must not have a {forbidden_field}.')
+
+
 class AnonymousAccess(models.Model):
     """
     Makes it possible to grant anonymous access (without user auth)