Allow pushing user-allocation membership to Keycloak

.github/workflows/test-functional-microshift.yaml

@@ -35,6 +35,10 @@ jobs:
         run: |
           bash ./ci/setup-oc-client.sh
 
+      - name: Install Keycloak
+        run: |
+          bash ./ci/setup-keycloak.sh
+
       - name: Install Microshift
         run: |
           ./ci/microshift.sh

.github/workflows/test-functional-microstack.yaml

@@ -18,6 +18,10 @@ jobs:
         with:
           python-version: 3.12
 
+      - name: Install Keycloak
+        run: |
+          bash ./ci/setup-keycloak.sh
+
       - name: Install ColdFront and plugin
         run: |
           python -m pip install --upgrade pip

ci/run_functional_tests_openshift.sh

@@ -5,6 +5,12 @@
 # Tests expect the resource to be name Devstack
 set -xe
 
+export KEYCLOAK_BASE_URL="http://localhost:8080"
+export KEYCLOAK_REALM="master"
+export KEYCLOAK_CLIENT_ID="admin-cli"
+export KEYCLOAK_ADMIN_USER="admin"
+export KEYCLOAK_ADMIN_PASSWORD="nomoresecret"
+
 export OPENSHIFT_MICROSHIFT_TOKEN="$(oc create token -n onboarding onboarding-serviceaccount)"
 export OPENSHIFT_MICROSHIFT_VERIFY="false"
 

ci/run_functional_tests_openstack.sh

@@ -5,6 +5,12 @@
 # Tests expect the resource to be name Devstack
 set -xe
 
+export KEYCLOAK_BASE_URL="http://localhost:8080"
+export KEYCLOAK_REALM="master"
+export KEYCLOAK_CLIENT_ID="admin-cli"
+export KEYCLOAK_ADMIN_USER="admin"
+export KEYCLOAK_ADMIN_PASSWORD="nomoresecret"
+
 export CREDENTIAL_NAME=$(openssl rand -base64 12)
 
 export OPENSTACK_DEVSTACK_APPLICATION_CREDENTIAL_SECRET=$(

ci/setup-keycloak.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+set -xe
+
+sudo docker run -d --name keycloak \
+    -e KEYCLOAK_ADMIN=admin \
+    -e KEYCLOAK_ADMIN_PASSWORD=nomoresecret \
+    -p 8080:8080 \
+    -p 8443:8443 \
+    quay.io/keycloak/keycloak:25.0 start-dev

requirements.txt

@@ -8,3 +8,4 @@ python-keystoneclient
 python-novaclient
 python-neutronclient
 python-swiftclient
+requests

src/coldfront_plugin_cloud/kc_client.py

@@ -0,0 +1,81 @@
+import os
+import functools
+
+import requests
+
+
+class KeyCloakAPIClient:
+    def __init__(self):
+        self.base_url = os.getenv("KEYCLOAK_BASE_URL")
+        self.realm = os.getenv("KEYCLOAK_REALM")
+        self.admin_user = os.getenv("KEYCLOAK_ADMIN_USER")
+        self.admin_password = os.getenv("KEYCLOAK_ADMIN_PASSWORD")
+        self.client_id = os.getenv("KEYCLOAK_CLIENT_ID", "admin-cli")
+
+        self.token_url = (
+            f"{self.base_url}/realms/{self.realm}/protocol/openid-connect/token"
+        )
+
+    @functools.cached_property
+    def api_client(self):
+        params = {
+            "grant_type": "password",
+            "client_id": self.client_id,
+            "username": self.admin_user,
+            "password": self.admin_password,
+            "scope": "openid",
+        }
+        r = requests.post(self.token_url, data=params).json()
+        headers = {
+            "Authorization": ("Bearer %s" % r["access_token"]),
+            "Content-Type": "application/json",
+        }
+        session = requests.session()
+        session.headers.update(headers)
+        return session
+
+    def create_group(self, group_name):
+        url = f"{self.base_url}/admin/realms/{self.realm}/groups"
+        payload = {"name": group_name}
+        response = self.api_client.post(url, json=payload)
+
+        # If group already exists, ignore and move on
+        if response.status_code not in (201, 409):
+            response.raise_for_status()
+
+    def create_user(self, cf_username):
+        """Helper function to create user in Keycloak, for testing purposes only"""
+        url = f"{self.base_url}/admin/realms/{self.realm}/users"
+        payload = {
+            "username": cf_username,
+            "enabled": True,
+            "email": cf_username,
+        }
+        r = self.api_client.post(url, json=payload)
+        r.raise_for_status()
+
+    def get_group_id(self, group_name) -> str | None:
+        """Return None if group not found"""
+        query = f"search={group_name}&exact=true"
+        url = f"{self.base_url}/admin/realms/{self.realm}/groups?{query}"
+        r = self.api_client.get(url).json()
+        return r[0]["id"] if r else None
+
+    def get_user_id(self, cf_username) -> str | None:
+        """Return None if user not found"""
+        # TODO (Quan): Confirm that Coldfront usernames map to Keycloak emails, not email, or something else?
+        query = f"email={cf_username}&exact=true"
+        url = f"{self.base_url}/admin/realms/{self.realm}/users?{query}"
+        r = self.api_client.get(url).json()
+        return r[0]["id"] if r else None
+
+    def add_user_to_group(self, user_id, group_id):
+        url = f"{self.base_url}/admin/realms/{self.realm}/users/{user_id}/groups/{group_id}"
+        r = self.api_client.put(url)
+        r.raise_for_status()
+
+    def get_user_groups(self, user_id) -> list[str]:
+        url = f"{self.base_url}/admin/realms/{self.realm}/users/{user_id}/groups"
+        r = self.api_client.get(url)
+        r.raise_for_status()
+        return [group["name"] for group in r.json()]

src/coldfront_plugin_cloud/tasks.py

@@ -12,6 +12,7 @@
     esi,
     openshift_vm,
     utils,
+    kc_client,
 )
 
 logger = logging.getLogger(__name__)
@@ -154,6 +155,17 @@ def set_quota_attributes():
 
         allocator.set_quota(project_id)
 
+        # After setting everything on cluster, add user to Keycloak group
+        kc_admin_client = kc_client.KeyCloakAPIClient()
+        kc_admin_client.create_group(project_id)
+        if user_id := kc_admin_client.get_user_id(pi_username):
+            group_id = kc_admin_client.get_group_id(project_id)
+            kc_admin_client.add_user_to_group(user_id, group_id)
+        else:
+            logger.warning(
+                f"User {pi_username} not found in Keycloak, cannot add to group."
+            )
+
 
 def disable_allocation(allocation_pk):
     allocation = Allocation.objects.get(pk=allocation_pk)

src/coldfront_plugin_cloud/tests/base.py

@@ -24,6 +24,8 @@
 from coldfront.core.field_of_science.models import FieldOfScience
 from django.core.management import call_command
 
+from coldfront_plugin_cloud import kc_client
+
 
 class TestBase(TestCase):
     def setUp(self) -> None:
@@ -37,11 +39,7 @@ def setUp(self) -> None:
         # For testing we can validate allocations with this status
         AllocationStatusChoice.objects.get_or_create(name="Active (Needs Renewal)")
 
-    @staticmethod
-    def new_user(username=None) -> User:
-        username = username or f"{uuid.uuid4().hex}@example.com"
-        User.objects.create(username=username, email=username)
-        return User.objects.get(username=username)
+        self.kc_admin_client = kc_client.KeyCloakAPIClient()
 
     @staticmethod
     def new_esi_resource(name=None, auth_url=None) -> Resource:
@@ -101,6 +99,15 @@ def new_openshift_resource(
         )
         return Resource.objects.get(name=resource_name)
 
+    def new_user(self, username=None, add_to_keycloak=True) -> User:
+        username = username or f"{uuid.uuid4().hex}@example.com"
+        User.objects.create(username=username, email=username)
+
+        if add_to_keycloak:
+            self.kc_admin_client.create_user(username)
+
+        return User.objects.get(username=username)
+
     def new_project(self, title=None, pi=None) -> Project:
         title = title or uuid.uuid4().hex
         pi = pi or self.new_user()

src/coldfront_plugin_cloud/tests/functional/openshift/test_allocation.py

@@ -52,6 +52,12 @@ def test_new_allocation(self):
 
         allocator._get_role(user.username, project_id)
 
+        # Check Keycloak group and user membership
+        self.kc_admin_client.get_group_id(project_id)
+        user_id = self.kc_admin_client.get_user_id(user.username)
+        assert project_id in self.kc_admin_client.get_user_groups(user_id)
+
+        # TODO (Quan): Confirm that user should also be removed from group on role removal
         allocator.remove_role_from_user(user.username, project_id)
 
         with self.assertRaises(openshift.NotFound):

src/coldfront_plugin_cloud/tests/unit/test_calculate_quota_unit_hours.py

@@ -22,7 +22,7 @@ def test_new_allocation_quota(self):
         )
 
         with freezegun.freeze_time("2020-03-15 00:01:00"):
-            user = self.new_user()
+            user = self.new_user(add_to_keycloak=False)
             project = self.new_project(pi=user)
             allocation = self.new_allocation(project, self.resource, 2)
             utils.set_attribute_on_allocation(
@@ -92,7 +92,7 @@ def test_new_allocation_quota_expired(self):
         self.resource = self.new_openshift_resource(
             name="",
         )
-        user = self.new_user()
+        user = self.new_user(add_to_keycloak=False)
         project = self.new_project(pi=user)
         allocation = self.new_allocation(project, self.resource, 2)
         allocation.status = allocation_models.AllocationStatusChoice.objects.get(
@@ -125,7 +125,7 @@ def test_new_allocation_quota_denied(self):
         self.resource = self.new_openshift_resource(
             name="",
         )
-        user = self.new_user()
+        user = self.new_user(add_to_keycloak=False)
         project = self.new_project(pi=user)
         allocation = self.new_allocation(project, self.resource, 2)
 
@@ -155,7 +155,7 @@ def test_new_allocation_quota_last_revoked(self):
         self.resource = self.new_openshift_resource(
             name="",
         )
-        user = self.new_user()
+        user = self.new_user(add_to_keycloak=False)
         project = self.new_project(pi=user)
         allocation = self.new_allocation(project, self.resource, 2)
 
@@ -200,7 +200,7 @@ def test_new_allocation_quota_new(self):
         self.resource = self.new_openshift_resource(
             name="",
         )
-        user = self.new_user()
+        user = self.new_user(add_to_keycloak=False)
         project = self.new_project(pi=user)
         allocation = self.new_allocation(project, self.resource, 2)
 
@@ -218,7 +218,7 @@ def test_new_allocation_quota_never_approved(self):
         self.resource = self.new_openshift_resource(
             name="",
         )
-        user = self.new_user()
+        user = self.new_user(add_to_keycloak=False)
         project = self.new_project(pi=user)
         allocation = self.new_allocation(project, self.resource, 2)
 
@@ -240,7 +240,7 @@ def test_change_request_decrease(self):
         self.resource = self.new_openshift_resource(
             name="",
         )
-        user = self.new_user()
+        user = self.new_user(add_to_keycloak=False)
         project = self.new_project(pi=user)
         allocation = self.new_allocation(project, self.resource, 2)
 
@@ -286,7 +286,7 @@ def test_change_request_increase(self):
         self.resource = self.new_openshift_resource(
             name="",
         )
-        user = self.new_user()
+        user = self.new_user(add_to_keycloak=False)
         project = self.new_project(pi=user)
         allocation = self.new_allocation(project, self.resource, 2)
 
@@ -332,7 +332,7 @@ def test_change_request_decrease_multiple(self):
         self.resource = self.new_openshift_resource(
             name="",
         )
-        user = self.new_user()
+        user = self.new_user(add_to_keycloak=False)
         project = self.new_project(pi=user)
         allocation = self.new_allocation(project, self.resource, 2)
 
@@ -395,7 +395,7 @@ def test_new_allocation_quota_change_request(self):
         self.resource = self.new_openshift_resource(
             name="",
         )
-        user = self.new_user()
+        user = self.new_user(add_to_keycloak=False)
         project = self.new_project(pi=user)
         allocation = self.new_allocation(project, self.resource, 2)
 

src/coldfront_plugin_cloud/tests/unit/test_migrate_field_of_science.py

@@ -18,9 +18,12 @@ def test_command_output(self):
         new_fos_1_des = uuid.uuid4().hex  # Migrate to new fos
         new_fos_2_des = old_fos_4.description  # Migrate to existing fos
 
-        fake_project_1 = self.new_project()
-        fake_project_2 = self.new_project()
-        fake_project_3 = self.new_project()
+        fake_user = self.new_user(
+            add_to_keycloak=False
+        )  # To avoid Keycloak dependency in unit test
+        fake_project_1 = self.new_project(pi=fake_user)
+        fake_project_2 = self.new_project(pi=fake_user)
+        fake_project_3 = self.new_project(pi=fake_user)
         fake_project_1.field_of_science = old_fos_1
         fake_project_2.field_of_science = old_fos_2
         fake_project_3.field_of_science = old_fos_3