From bf025ef50fb76cc5e19863563049264a0c1ad2bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Benguigui?= Date: Fri, 28 Feb 2025 09:13:56 +0100 Subject: [PATCH 1/4] GCE integration --- .../sal/model/CloudProviderType.java | 2 +- .../sal/model/EmsDeploymentRequest.java | 2 +- .../sal/service/nc/NodeCandidateUtils.java | 7 +- .../sal/service/service/NodeService.java | 11 ++ .../PAConnectorIaasGateway.java | 6 + .../src/main/resources/Define_NS_GCE.xml | 120 ++++++++++++++++++ 6 files changed, 144 insertions(+), 4 deletions(-) create mode 100644 sal-service/src/main/resources/Define_NS_GCE.xml diff --git a/sal-common/src/main/java/org/ow2/proactive/sal/model/CloudProviderType.java b/sal-common/src/main/java/org/ow2/proactive/sal/model/CloudProviderType.java index f3d6d86f..fcf68cf9 100644 --- a/sal-common/src/main/java/org/ow2/proactive/sal/model/CloudProviderType.java +++ b/sal-common/src/main/java/org/ow2/proactive/sal/model/CloudProviderType.java @@ -16,7 +16,7 @@ public enum CloudProviderType { AWS_EC2("aws-ec2"), AZURE("azure"), - GCE("gce"), + GCE("google-compute-engine"), OPENSTACK("openstack"), //openstack-nova EDGE("EDGE"), diff --git a/sal-common/src/main/java/org/ow2/proactive/sal/model/EmsDeploymentRequest.java b/sal-common/src/main/java/org/ow2/proactive/sal/model/EmsDeploymentRequest.java index 9115a67c..e7f7e2f3 100644 --- a/sal-common/src/main/java/org/ow2/proactive/sal/model/EmsDeploymentRequest.java +++ b/sal-common/src/main/java/org/ow2/proactive/sal/model/EmsDeploymentRequest.java @@ -59,7 +59,7 @@ public enum TargetProvider { // Azure VM AZUREVM("azure"), // Google CLoud Engine - GCE("gce"), + GCE("google-compute-engine"), // OpenStack NOVA OPENSTACKNOVA("openstack"), // BYON, to be used for on-premise baremetal diff --git a/sal-service/src/main/java/org/ow2/proactive/sal/service/nc/NodeCandidateUtils.java b/sal-service/src/main/java/org/ow2/proactive/sal/service/nc/NodeCandidateUtils.java index 31cfa82b..783aa8d7 100644 --- a/sal-service/src/main/java/org/ow2/proactive/sal/service/nc/NodeCandidateUtils.java +++ b/sal-service/src/main/java/org/ow2/proactive/sal/service/nc/NodeCandidateUtils.java @@ -381,14 +381,14 @@ public void saveNodeCandidates(List newCloudIds) { List consolidatedImages = images.toList() .parallelStream() .map(NodeCandidateUtils::convertObjectToJson) - .filter(record -> !blacklistedRegions.contains(record.get("location"))) + .filter(record -> record.get("location").toString().isEmpty() || + !blacklistedRegions.contains(record.get("location"))) .collect(Collectors.toList()); LOGGER.info("Consolidated images: {}", consolidatedImages); //TODO: (Optimization) An images per region map structure could be the best here. // It can reduce the getNodeCandidates calls to PA. List entries = new LinkedList<>(); - List openstackOsList = Arrays.asList("Ubuntu", "Fedora", "Centos", "Debian"); consolidatedImages.forEach(image -> { String region = image.optString("location"); String imageReq; @@ -406,6 +406,9 @@ public void saveNodeCandidates(List newCloudIds) { case AZURE: imageReq = os; break; + case GCE: + imageReq = os; + break; default: throw new IllegalArgumentException("The infrastructure " + paCloud.getCloudType() + " is not handled yet."); diff --git a/sal-service/src/main/java/org/ow2/proactive/sal/service/service/NodeService.java b/sal-service/src/main/java/org/ow2/proactive/sal/service/service/NodeService.java index b1ecd125..9d92e9ec 100644 --- a/sal-service/src/main/java/org/ow2/proactive/sal/service/service/NodeService.java +++ b/sal-service/src/main/java/org/ow2/proactive/sal/service/service/NodeService.java @@ -251,6 +251,17 @@ private void defineNSWithDeploymentInfo(String nodeSourceName, PACloud cloud, De variables.put("vmPublicKey", cloud.getSshCredentials().getPublicKey()); variables.put("region", deployment.getNode().getNodeCandidate().getLocation().getName()); break; + case GCE: + filename = File.separator + "Define_NS_GCE.xml"; + variables.put("vmUsername", cloud.getSshCredentials().getUsername()); + variables.put("vmPublicKey", cloud.getSshCredentials().getPublicKey()); + variables.put("vmPrivateKey", cloud.getSshCredentials().getPrivateKey()); + variables.put("image", deployment.getNode().getNodeCandidate().getImage().getId()); + variables.put("region", deployment.getNode().getNodeCandidate().getLocation().getName()); + variables.put("machineType", deployment.getNode().getNodeCandidate().getHardware().getProviderId()); + variables.put("ram", deployment.getNode().getNodeCandidate().getHardware().getRam().toString()); + variables.put("cores", deployment.getNode().getNodeCandidate().getHardware().getCores().toString()); + break; default: throw new IllegalArgumentException("Unhandled cloud provider: " + cloud.getCloudProvider()); } diff --git a/sal-service/src/main/java/org/ow2/proactive/sal/service/service/infrastructure/PAConnectorIaasGateway.java b/sal-service/src/main/java/org/ow2/proactive/sal/service/service/infrastructure/PAConnectorIaasGateway.java index 4ef10165..d570183a 100644 --- a/sal-service/src/main/java/org/ow2/proactive/sal/service/service/infrastructure/PAConnectorIaasGateway.java +++ b/sal-service/src/main/java/org/ow2/proactive/sal/service/service/infrastructure/PAConnectorIaasGateway.java @@ -152,6 +152,12 @@ public void defineInfrastructure(String infrastructureName, PACloud cloud, Strin cloud.getCredentials().getDomain() + "\", \"subscriptionId\": \"" + cloud.getCredentials().getSubscriptionId() + "\"}}"; break; + case GCE: + jsonOutputString = "{\"id\": \"" + infrastructureName + "\"," + "\"type\": \"" + + cloud.getCloudProvider() + "\"," + "\"credentials\": {\"username\": \"" + + cloud.getCredentials().getUserName() + "\", \"password\": \"" + + cloud.getCredentials().getPrivateKey() + "\"}, \"region\": \"" + region + "\"}"; + break; default: throw new IllegalArgumentException("The infrastructure " + infrastructureName + " is not handled yet."); } diff --git a/sal-service/src/main/resources/Define_NS_GCE.xml b/sal-service/src/main/resources/Define_NS_GCE.xml new file mode 100644 index 00000000..8561900e --- /dev/null +++ b/sal-service/src/main/resources/Define_NS_GCE.xml @@ -0,0 +1,120 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 142.125 + + + 351.5 + + + + + + + + + + + + + + + + ]]> + + + \ No newline at end of file From 278bb2a1d7d05b026c1e33ab0a204ac1a39f776b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Benguigui?= Date: Tue, 11 Mar 2025 11:44:34 +0100 Subject: [PATCH 2/4] GCE integration --- .../ow2/proactive/sal/model/Credential.java | 5 + .../ow2/proactive/sal/model/Credentials.java | 6 + .../sal/service/nc/NodeCandidateUtils.java | 119 ++++++++++-------- .../sal/service/service/CloudService.java | 4 + .../sal/service/service/NodeService.java | 3 + .../PAConnectorIaasGateway.java | 3 +- .../src/main/resources/Define_NS_GCE.xml | 19 ++- 7 files changed, 97 insertions(+), 62 deletions(-) diff --git a/sal-common/src/main/java/org/ow2/proactive/sal/model/Credential.java b/sal-common/src/main/java/org/ow2/proactive/sal/model/Credential.java index 865b0c08..10c9456f 100644 --- a/sal-common/src/main/java/org/ow2/proactive/sal/model/Credential.java +++ b/sal-common/src/main/java/org/ow2/proactive/sal/model/Credential.java @@ -28,6 +28,8 @@ public class Credential implements Serializable { // JSON field constants public static final String JSON_USER = "user"; + public static final String JSON_PROJECT_ID = "projectId"; + public static final String JSON_SECRET = "secret"; public static final String JSON_DOMAIN = "domain"; @@ -37,6 +39,9 @@ public class Credential implements Serializable { @JsonProperty(JSON_USER) private String user = null; + @JsonProperty(JSON_PROJECT_ID) + private String projectId = null; + @JsonProperty(JSON_SECRET) private String secret = null; diff --git a/sal-common/src/main/java/org/ow2/proactive/sal/model/Credentials.java b/sal-common/src/main/java/org/ow2/proactive/sal/model/Credentials.java index 033a1135..7dfe23a3 100644 --- a/sal-common/src/main/java/org/ow2/proactive/sal/model/Credentials.java +++ b/sal-common/src/main/java/org/ow2/proactive/sal/model/Credentials.java @@ -31,6 +31,8 @@ public class Credentials implements Serializable { public static final String JSON_USER_NAME = "userName"; + public static final String JSON_PROJECT_ID = "projectId"; + public static final String JSON_PASSWORD = "password"; public static final String JSON_PRIVATE_KEY = "privateKey"; @@ -51,6 +53,10 @@ public class Credentials implements Serializable { @JsonProperty(JSON_USER_NAME) private String userName; + @Column(name = "PROJECT_ID") + @JsonProperty(JSON_PROJECT_ID) + private String projectId; + @Column(name = "PASSWORD") @JsonProperty(JSON_PASSWORD) private String password; diff --git a/sal-service/src/main/java/org/ow2/proactive/sal/service/nc/NodeCandidateUtils.java b/sal-service/src/main/java/org/ow2/proactive/sal/service/nc/NodeCandidateUtils.java index 783aa8d7..537d12d3 100644 --- a/sal-service/src/main/java/org/ow2/proactive/sal/service/nc/NodeCandidateUtils.java +++ b/sal-service/src/main/java/org/ow2/proactive/sal/service/nc/NodeCandidateUtils.java @@ -18,6 +18,7 @@ import javax.annotation.PostConstruct; import org.apache.commons.lang3.StringUtils; +import org.javatuples.Pair; import org.javatuples.Quartet; import org.json.JSONArray; import org.json.JSONObject; @@ -363,6 +364,22 @@ private static JSONObject convertObjectToJson(Object object) { return myJson; } + private String getOsAccordingToCloudProvider(CloudProviderType cloudProviderType, String os) + throws IllegalArgumentException { + switch (cloudProviderType) { + case AWS_EC2: + return "Linux"; + case OPENSTACK: + return os; + case AZURE: + return os; + case GCE: + return os; + default: + throw new IllegalArgumentException("The infrastructure " + cloudProviderType + " is not handled yet."); + } + } + public void saveNodeCandidates(List newCloudIds) { newCloudIds.forEach(newCloudId -> { PACloud paCloud = repositoryService.getPACloud(newCloudId); @@ -378,68 +395,60 @@ public void saveNodeCandidates(List newCloudIds) { return; } LOGGER.info("Returned images: {}", images); - List consolidatedImages = images.toList() - .parallelStream() - .map(NodeCandidateUtils::convertObjectToJson) - .filter(record -> record.get("location").toString().isEmpty() || - !blacklistedRegions.contains(record.get("location"))) - .collect(Collectors.toList()); - LOGGER.info("Consolidated images: {}", consolidatedImages); - - //TODO: (Optimization) An images per region map structure could be the best here. - // It can reduce the getNodeCandidates calls to PA. - List entries = new LinkedList<>(); - consolidatedImages.forEach(image -> { - String region = image.optString("location"); - String imageReq; - String os = (String) ((JSONObject) image.get("operatingSystem")).get("family"); - os = os.substring(0, 1).toUpperCase() + os.substring(1); - String pair = os + ":" + region; - - switch (paCloud.getCloudProvider()) { - case AWS_EC2: - imageReq = "Linux"; - break; - case OPENSTACK: - imageReq = os; - break; - case AZURE: - imageReq = os; - break; - case GCE: - imageReq = os; - break; - default: - throw new IllegalArgumentException("The infrastructure " + paCloud.getCloudType() + - " is not handled yet."); - } - - if (paCloud.getCloudProvider() == OPENSTACK) { - entries.add(pair); + Map, List> consolidatedImagesGrouped = images.toList() + .parallelStream() + .map(NodeCandidateUtils::convertObjectToJson) + .filter(record -> record.get("location") + .toString() + .isEmpty() || + !blacklistedRegions.contains(record.get("location"))) + .collect(Collectors.groupingBy(image -> { + // Retrieve the region + String region = image.optString("location"); + // Retrieve the imageReq + String os = (String) ((JSONObject) image.get("operatingSystem")).get("family"); + os = os.substring(0, 1) + .toUpperCase() + + os.substring(1); + String imageReq = getOsAccordingToCloudProvider(paCloud.getCloudProvider(), + os); + + return new Pair<>(region, + imageReq); + })); + + consolidatedImagesGrouped.entrySet().parallelStream().forEach(entry -> { + String region = entry.getKey().getValue0(); + String imageReq = entry.getKey().getValue1(); + List imageList = entry.getValue(); + + try { + JSONArray nodeCandidates = nodeCandidatesCache.get(Quartet.with(paCloud, region, imageReq, "")); + + imageList.forEach(image -> createAndStoreIaasNodesAndNodeCandidates(nodeCandidates, + paCloud, + image)); + } catch (ExecutionException e) { + LOGGER.error("Could not get node candidates from cache: ", e); } - populateNodeCandidatesFromCache(paCloud, region, imageReq, image); }); + }); repositoryService.flush(); } - private void populateNodeCandidatesFromCache(PACloud paCloud, String region, String imageReq, JSONObject image) { - try { - JSONArray nodeCandidates = nodeCandidatesCache.get(Quartet.with(paCloud, region, imageReq, "")); - nodeCandidates.forEach(nc -> { - JSONObject nodeCandidate = (JSONObject) nc; - createLocation(nodeCandidate, paCloud); - NodeCandidate newNodeCandidate = createNodeCandidate(nodeCandidate, image, paCloud); - repositoryService.saveNodeCandidate(newNodeCandidate); - IaasNode newIaasNode = new IaasNode(newNodeCandidate); - repositoryService.saveIaasNode(newIaasNode); - newNodeCandidate.setNodeId(newIaasNode.getId()); - repositoryService.saveNodeCandidate(newNodeCandidate); - }); - } catch (ExecutionException ee) { - LOGGER.error("Could not get node candidates from cache: ", ee); - } + private void createAndStoreIaasNodesAndNodeCandidates(JSONArray nodeCandidates, PACloud paCloud, JSONObject image) { + nodeCandidates.forEach(nc -> { + JSONObject nodeCandidate = (JSONObject) nc; + createLocation(nodeCandidate, paCloud); + NodeCandidate newNodeCandidate = createNodeCandidate(nodeCandidate, image, paCloud); + repositoryService.saveNodeCandidate(newNodeCandidate); + IaasNode newIaasNode = new IaasNode(newNodeCandidate); + repositoryService.saveIaasNode(newIaasNode); + newNodeCandidate.setNodeId(newIaasNode.getId()); + repositoryService.saveNodeCandidate(newNodeCandidate); + }); } private JSONArray getAllPagedNodeCandidates(PACloud paCloud, String region, String imageReq, String token) { diff --git a/sal-service/src/main/java/org/ow2/proactive/sal/service/service/CloudService.java b/sal-service/src/main/java/org/ow2/proactive/sal/service/service/CloudService.java index 46742039..550fcfee 100644 --- a/sal-service/src/main/java/org/ow2/proactive/sal/service/service/CloudService.java +++ b/sal-service/src/main/java/org/ow2/proactive/sal/service/service/CloudService.java @@ -92,6 +92,7 @@ public Integer addClouds(String sessionId, List clouds) throws Credentials credentials = new Credentials(); credentials.setUserName(cloud.getCredentials().getUser()); + credentials.setProjectId(cloud.getCredentials().getProjectId()); credentials.setPrivateKey(cloud.getCredentials().getSecret()); credentials.setDomain(cloud.getCredentials().getDomain()); credentials.setSubscriptionId(cloud.getCredentials().getSubscriptionId()); @@ -453,6 +454,9 @@ private Credentials hideCredentials(Credentials creds) { if (creds.getUserName() != null) { newCreds.setUserName(hideString(creds.getUserName(), 5)); } + if (creds.getProjectId() != null) { + newCreds.setProjectId(hideString(creds.getProjectId(), 5)); + } } return newCreds; } diff --git a/sal-service/src/main/java/org/ow2/proactive/sal/service/service/NodeService.java b/sal-service/src/main/java/org/ow2/proactive/sal/service/service/NodeService.java index 9d92e9ec..86d2fa32 100644 --- a/sal-service/src/main/java/org/ow2/proactive/sal/service/service/NodeService.java +++ b/sal-service/src/main/java/org/ow2/proactive/sal/service/service/NodeService.java @@ -253,6 +253,9 @@ private void defineNSWithDeploymentInfo(String nodeSourceName, PACloud cloud, De break; case GCE: filename = File.separator + "Define_NS_GCE.xml"; + variables.put("user", cloud.getCredentials().getUserName()); + variables.put("projectId", cloud.getCredentials().getProjectId()); + variables.put("secret", cloud.getCredentials().getPrivateKey()); variables.put("vmUsername", cloud.getSshCredentials().getUsername()); variables.put("vmPublicKey", cloud.getSshCredentials().getPublicKey()); variables.put("vmPrivateKey", cloud.getSshCredentials().getPrivateKey()); diff --git a/sal-service/src/main/java/org/ow2/proactive/sal/service/service/infrastructure/PAConnectorIaasGateway.java b/sal-service/src/main/java/org/ow2/proactive/sal/service/service/infrastructure/PAConnectorIaasGateway.java index d570183a..98a4c016 100644 --- a/sal-service/src/main/java/org/ow2/proactive/sal/service/service/infrastructure/PAConnectorIaasGateway.java +++ b/sal-service/src/main/java/org/ow2/proactive/sal/service/service/infrastructure/PAConnectorIaasGateway.java @@ -155,7 +155,8 @@ public void defineInfrastructure(String infrastructureName, PACloud cloud, Strin case GCE: jsonOutputString = "{\"id\": \"" + infrastructureName + "\"," + "\"type\": \"" + cloud.getCloudProvider() + "\"," + "\"credentials\": {\"username\": \"" + - cloud.getCredentials().getUserName() + "\", \"password\": \"" + + cloud.getCredentials().getUserName() + "\", \"projectId\": \"" + + cloud.getCredentials().getProjectId() + "\", \"password\": \"" + cloud.getCredentials().getPrivateKey() + "\"}, \"region\": \"" + region + "\"}"; break; default: diff --git a/sal-service/src/main/resources/Define_NS_GCE.xml b/sal-service/src/main/resources/Define_NS_GCE.xml index 8561900e..89d20d22 100644 --- a/sal-service/src/main/resources/Define_NS_GCE.xml +++ b/sal-service/src/main/resources/Define_NS_GCE.xml @@ -3,6 +3,9 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="urn:proactive:jobdescriptor:3.13" xsi:schemaLocation="urn:proactive:jobdescriptor:3.13 http://www.activeeon.com/public_content/schemas/proactive/jobdescriptor/3.13/schedulerjob.xsd" name="Define_NS_GCE" projectName="NebulOuS" priority="normal" onTaskError="continueJobExecution" maxNumberOfExecution="2" > + + + @@ -14,6 +17,7 @@ + @@ -26,7 +30,7 @@ fork="true"> - +