k3s introduction (#127)

* labelNodes k3s

* application deployment update

* wait for master update

* Update docker-compose env var

* documentation and deployment update

Author: ankicabarisic

README.md

@@ -98,6 +98,13 @@ docker pull activeeon/sal:dev
 
 * Open [docker-compose.yaml](https://github.com/ow2-proactive/scheduling-abstraction-layer/blob/master/docker/docker-compose.yaml)
 
+* Specify which version of the Kubernetes cluster you will deploy. Default scripts set for deployment are for k8s.
+
+
+```bash
+    CLUSTER_TYPE: "k8s"  # (defult) or "k3s"
+```
+
 * Setup connection to the ProActive scheduler
 
 ```bash

docker/docker-compose.yaml

@@ -41,6 +41,7 @@ services:
       - ./wait_for_db.sh:/usr/local/tomcat/bin/wait_for_db.sh
     environment:
       PROPERTIES_FILENAME: sal
+      CLUSTER_TYPE: "k8s"  # (defult) or "k3s" 
       #Set up connection to ProActive server (PWS)
       PWS_URL: <CHANGE_ME>
       PWS_USERNAME: <CHANGE_ME>

docker/kubernetes/sal.yaml

@@ -64,6 +64,8 @@ spec:
           env:
             - name: MYSQL_DATABASE
               value: proactive
+            - name: CLUSTER_TYPE
+              value: "k8s"  # Change to "k3s" when needed
             - name: PROPERTIES_FILENAME
               value: sal
             - name: PWS_URL

sal-service/src/main/java/org/ow2/proactive/sal/service/service/TaskBuilder.java

@@ -17,6 +17,7 @@
 import org.ow2.proactive.sal.service.nc.WhiteListedInstanceTypesUtils;
 import org.ow2.proactive.sal.service.service.application.PAFactory;
 import org.ow2.proactive.sal.service.util.ByonUtils;
+import org.ow2.proactive.sal.service.util.ClusterUtils;
 import org.ow2.proactive.sal.service.util.Utils;
 import org.ow2.proactive.scheduler.common.task.ScriptTask;
 import org.ow2.proactive.scheduler.common.task.TaskVariable;
@@ -78,6 +79,8 @@ public class TaskBuilder {
 
     private static final String WAIT_FOR_MASTER_SCRIPT = "wait_for_master.groovy";
 
+    private static final String WAIT_FOR_MASTER_K3S_SCRIPT = "wait_for_master_k3s.groovy";
+
     private static final String SET_TOKEN_SCRIPT = "set_token.groovy";
 
     private static final String DELETE_NODE_SCRIPT = "delete_node.groovy";
@@ -899,9 +902,18 @@ private Map<String, TaskVariable> createVariablesMapForSynchronizationChannels(J
     }
 
     private ScriptTask createWaitForMasterTask(String masterNodeToken) {
-        ScriptTask waitForMasterTask = PAFactory.createGroovyScriptTaskFromFile("wait_for_master",
-                                                                                WAIT_FOR_MASTER_SCRIPT);
+        String clusterType = System.getenv(ClusterUtils.CLUSTER_TYPE_ENV);
+        String waitForMasterScript;
+
+        if (ClusterUtils.CLUSTER_TYPE_K3S.equalsIgnoreCase(clusterType)) {
+            waitForMasterScript = WAIT_FOR_MASTER_K3S_SCRIPT;
+        } else {
+            waitForMasterScript = WAIT_FOR_MASTER_SCRIPT;
+        }
+
+        ScriptTask waitForMasterTask = PAFactory.createGroovyScriptTaskFromFile("wait_for_master", waitForMasterScript);
         waitForMasterTask.addGenericInformation("NODE_ACCESS_TOKEN", masterNodeToken);
+
         return waitForMasterTask;
     }
 

sal-service/src/main/java/org/ow2/proactive/sal/service/util/ClusterUtils.java

@@ -24,6 +24,12 @@
 @Log4j2
 public class ClusterUtils {
 
+    public static final String CLUSTER_TYPE_ENV = "CLUSTER_TYPE";
+
+    public static final String CLUSTER_TYPE_K3S = "k3s";
+
+    public static final String CLUSTER_TYPE_K8S = "k8s";
+
     private static final String SCRIPTS_PATH = "/usr/local/tomcat/scripts/";
 
     // TO be changed, the hardcoding of the ubuntu user is a bad practice.
@@ -33,6 +39,13 @@ public class ClusterUtils {
 
     private static final String FILE_PATH = "/home/ubuntu/.profile";
 
+    // K3s-related commands
+    private static final String CLI_K3s_USER_SELECTION = "$dau bash -c";
+
+    private static final String K3S_COMMANDS = "dau=\"sudo -H -E -u ubuntu\"\n" +
+                                               "export KUBECONFIG=/etc/rancher/k3s/k3s.yaml\n" +
+                                               "echo \"KUBECONFIG=${KUBECONFIG}\" | sudo tee -a /etc/environment\n";
+
     public static Job createMasterNodeJob(String clusterName, ClusterNodeDefinition masterNode, PACloud cloud,
             String envVars) throws IOException {
         Job masterNodeJob = new Job();
@@ -174,6 +187,33 @@ private static String getBashFilesContent(String fileName) throws IOException {
     }
 
     public static String createLabelNodesScript(List<Map<String, String>> nodeLabels, String clusterName) {
+        String clusterType = System.getenv(CLUSTER_TYPE_ENV);
+
+        if (CLUSTER_TYPE_K3S.equalsIgnoreCase(clusterType)) {
+            return createK3sLabelNodesScript(nodeLabels, clusterName);
+        } else {
+            return createK8sLabelNodesScript(nodeLabels, clusterName);
+        }
+    }
+
+    public static String createK3sLabelNodesScript(List<Map<String, String>> nodeLabels, String clusterName) {
+        StringBuilder script = new StringBuilder();
+        script.append(K3S_COMMANDS).append("\n");
+        for (Map<String, String> nodeLabelPair : nodeLabels) {
+            for (String nodeName : nodeLabelPair.keySet()) {
+                String label = nodeLabelPair.get(nodeName);
+                script.append(String.format("%s '%s %s-%s %s' \n",
+                                            CLI_K3s_USER_SELECTION,
+                                            KUBE_LABEL_COMMAND,
+                                            nodeName.toLowerCase(),
+                                            clusterName,
+                                            label));
+            }
+        }
+        return script.toString();
+    }
+
+    public static String createK8sLabelNodesScript(List<Map<String, String>> nodeLabels, String clusterName) {
         StringBuilder script = new StringBuilder();
         for (Map<String, String> nodeLabelPair : nodeLabels) {
             for (String nodeName : nodeLabelPair.keySet()) {
@@ -190,34 +230,46 @@ public static String createLabelNodesScript(List<Map<String, String>> nodeLabels
     }
 
     public static String createDeployApplicationScript(ClusterApplication application) throws IOException {
+        String clusterType = System.getenv(CLUSTER_TYPE_ENV); // Get cluster type from env variable
+        return createDeployApplicationScript(application, clusterType);
+    }
+
+    public static String createDeployApplicationScript(ClusterApplication application, String clusterType)
+            throws IOException {
         String fileName = "/home/ubuntu/" + application.getAppName() + ".yaml";
         application.setYamlManager(ClusterApplication.PackageManagerEnum.getPackageManagerEnumByName(application.getPackageManager()));
-        String appCommand = createAppCommand(application.getYamlManager(), fileName);
+        String appCommand = createAppCommand(application.getYamlManager(), fileName, clusterType);
 
         if (appCommand == null) {
             LOGGER.error("\"{}\" is not supported!", application.getPackageManager());
             throw new IOException("yaml executor is not supported!");
         }
-        BufferedReader bufReader = new BufferedReader(new StringReader(application.getAppFile()));
+
         StringBuilder script = new StringBuilder();
-        String line = null;
-        script.append("sudo rm -f " + fileName + " || echo 'file was not found.' \n");
+        script.append("sudo rm -f ").append(fileName).append(" || echo 'file was not found.' \n");
 
-        // start heredoc
+        // Start heredoc
         script.append("cat <<'EOF' >").append(fileName).append("\n");
-        // embed the application YAML directly
-        script.append(application.getAppFile());
-        // end heredoc
+        script.append(application.getAppFile()); // Embed the YAML file content
         script.append("\nEOF\n");
 
-        script.append("sudo chown ubuntu:ubuntu " + fileName + "\n");
+        script.append("sudo chown ubuntu:ubuntu ").append(fileName).append("\n");
+
+        // Insert K3s-specific commands if needed
+        if (CLUSTER_TYPE_K3S.equalsIgnoreCase(clusterType)) {
+            script.append(K3S_COMMANDS).append("\n");
+        }
+
         script.append(appCommand);
         return script.toString();
     }
 
-    private static String createAppCommand(ClusterApplication.PackageManagerEnum yamlManager, String fileName) {
+    private static String createAppCommand(ClusterApplication.PackageManagerEnum yamlManager, String fileName,
+            String clusterType) {
         if (yamlManager != null) {
-            return String.format("%s '%s %s'", CLI_USER_SELECTION, yamlManager.getCommand(), fileName);
+            String cliSelection = CLUSTER_TYPE_K3S.equalsIgnoreCase(clusterType) ? CLI_K3s_USER_SELECTION
+                                                                                 : CLI_USER_SELECTION;
+            return String.format("%s '%s %s'", cliSelection, yamlManager.getCommand(), fileName);
         } else {
             LOGGER.error("The selected yaml executor is not supported!");
             return null;

sal-service/src/main/resources/wait_for_master_k3s.groovy

@@ -0,0 +1,41 @@
+import groovy.transform.Synchronized
+
+// Define a function to retrieve the IP from a server
+def retrieveIPFromServer(server) {
+    try {
+        def url = "http://${server}"
+        def response = new URL(url).text
+        return response.trim()
+    } catch (Exception e) {
+        println "Failed to retrieve IP from ${server}: ${e.message}"
+        return null
+    }
+}
+
+// Define a function to check the IP from multiple servers
+@Synchronized
+def checkIPFromServers() {
+    def servers = [
+            "checkip.amazonaws.com",
+            "api.ipify.org",
+            "ifconfig.me",
+            "ipinfo.io/ip",
+            "icanhazip.com",
+            "ident.me",
+            "myip.dnsomatic.com"
+    ];
+    for (def server : servers) {
+        def ip = retrieveIPFromServer(server)
+        if (ip) {
+            println "Address was retrieved from ${server}"
+            println "Public IP: ${ip}"
+            variables.put("masterIp", ip)
+            return ip
+        }
+    }
+
+    println "Unable to retrieve the public IP from any server."
+}
+
+// Call the function to check the IP
+result = checkIPFromServers()