diff --git a/sal-common/src/main/java/org/ow2/proactive/sal/model/CloudProviderType.java b/sal-common/src/main/java/org/ow2/proactive/sal/model/CloudProviderType.java index f3d6d86..fcf68cf 100644 --- a/sal-common/src/main/java/org/ow2/proactive/sal/model/CloudProviderType.java +++ b/sal-common/src/main/java/org/ow2/proactive/sal/model/CloudProviderType.java @@ -16,7 +16,7 @@ public enum CloudProviderType { AWS_EC2("aws-ec2"), AZURE("azure"), - GCE("gce"), + GCE("google-compute-engine"), OPENSTACK("openstack"), //openstack-nova EDGE("EDGE"), diff --git a/sal-common/src/main/java/org/ow2/proactive/sal/model/Credential.java b/sal-common/src/main/java/org/ow2/proactive/sal/model/Credential.java index 865b0c0..10c9456 100644 --- a/sal-common/src/main/java/org/ow2/proactive/sal/model/Credential.java +++ b/sal-common/src/main/java/org/ow2/proactive/sal/model/Credential.java @@ -28,6 +28,8 @@ public class Credential implements Serializable { // JSON field constants public static final String JSON_USER = "user"; + public static final String JSON_PROJECT_ID = "projectId"; + public static final String JSON_SECRET = "secret"; public static final String JSON_DOMAIN = "domain"; @@ -37,6 +39,9 @@ public class Credential implements Serializable { @JsonProperty(JSON_USER) private String user = null; + @JsonProperty(JSON_PROJECT_ID) + private String projectId = null; + @JsonProperty(JSON_SECRET) private String secret = null; diff --git a/sal-common/src/main/java/org/ow2/proactive/sal/model/Credentials.java b/sal-common/src/main/java/org/ow2/proactive/sal/model/Credentials.java index 033a113..7dfe23a 100644 --- a/sal-common/src/main/java/org/ow2/proactive/sal/model/Credentials.java +++ b/sal-common/src/main/java/org/ow2/proactive/sal/model/Credentials.java @@ -31,6 +31,8 @@ public class Credentials implements Serializable { public static final String JSON_USER_NAME = "userName"; + public static final String JSON_PROJECT_ID = "projectId"; + public static final String JSON_PASSWORD = "password"; public static final String JSON_PRIVATE_KEY = "privateKey"; @@ -51,6 +53,10 @@ public class Credentials implements Serializable { @JsonProperty(JSON_USER_NAME) private String userName; + @Column(name = "PROJECT_ID") + @JsonProperty(JSON_PROJECT_ID) + private String projectId; + @Column(name = "PASSWORD") @JsonProperty(JSON_PASSWORD) private String password; diff --git a/sal-common/src/main/java/org/ow2/proactive/sal/model/EmsDeploymentRequest.java b/sal-common/src/main/java/org/ow2/proactive/sal/model/EmsDeploymentRequest.java index 9115a67..e7f7e2f 100644 --- a/sal-common/src/main/java/org/ow2/proactive/sal/model/EmsDeploymentRequest.java +++ b/sal-common/src/main/java/org/ow2/proactive/sal/model/EmsDeploymentRequest.java @@ -59,7 +59,7 @@ public enum TargetProvider { // Azure VM AZUREVM("azure"), // Google CLoud Engine - GCE("gce"), + GCE("google-compute-engine"), // OpenStack NOVA OPENSTACKNOVA("openstack"), // BYON, to be used for on-premise baremetal diff --git a/sal-service/src/main/java/org/ow2/proactive/sal/service/nc/NodeCandidateUtils.java b/sal-service/src/main/java/org/ow2/proactive/sal/service/nc/NodeCandidateUtils.java index 31cfa82..537d12d 100644 --- a/sal-service/src/main/java/org/ow2/proactive/sal/service/nc/NodeCandidateUtils.java +++ b/sal-service/src/main/java/org/ow2/proactive/sal/service/nc/NodeCandidateUtils.java @@ -18,6 +18,7 @@ import javax.annotation.PostConstruct; import org.apache.commons.lang3.StringUtils; +import org.javatuples.Pair; import org.javatuples.Quartet; import org.json.JSONArray; import org.json.JSONObject; @@ -363,6 +364,22 @@ private static JSONObject convertObjectToJson(Object object) { return myJson; } + private String getOsAccordingToCloudProvider(CloudProviderType cloudProviderType, String os) + throws IllegalArgumentException { + switch (cloudProviderType) { + case AWS_EC2: + return "Linux"; + case OPENSTACK: + return os; + case AZURE: + return os; + case GCE: + return os; + default: + throw new IllegalArgumentException("The infrastructure " + cloudProviderType + " is not handled yet."); + } + } + public void saveNodeCandidates(List newCloudIds) { newCloudIds.forEach(newCloudId -> { PACloud paCloud = repositoryService.getPACloud(newCloudId); @@ -378,65 +395,60 @@ public void saveNodeCandidates(List newCloudIds) { return; } LOGGER.info("Returned images: {}", images); - List consolidatedImages = images.toList() - .parallelStream() - .map(NodeCandidateUtils::convertObjectToJson) - .filter(record -> !blacklistedRegions.contains(record.get("location"))) - .collect(Collectors.toList()); - LOGGER.info("Consolidated images: {}", consolidatedImages); - - //TODO: (Optimization) An images per region map structure could be the best here. - // It can reduce the getNodeCandidates calls to PA. - List entries = new LinkedList<>(); - List openstackOsList = Arrays.asList("Ubuntu", "Fedora", "Centos", "Debian"); - consolidatedImages.forEach(image -> { - String region = image.optString("location"); - String imageReq; - String os = (String) ((JSONObject) image.get("operatingSystem")).get("family"); - os = os.substring(0, 1).toUpperCase() + os.substring(1); - String pair = os + ":" + region; - - switch (paCloud.getCloudProvider()) { - case AWS_EC2: - imageReq = "Linux"; - break; - case OPENSTACK: - imageReq = os; - break; - case AZURE: - imageReq = os; - break; - default: - throw new IllegalArgumentException("The infrastructure " + paCloud.getCloudType() + - " is not handled yet."); - } - - if (paCloud.getCloudProvider() == OPENSTACK) { - entries.add(pair); + Map, List> consolidatedImagesGrouped = images.toList() + .parallelStream() + .map(NodeCandidateUtils::convertObjectToJson) + .filter(record -> record.get("location") + .toString() + .isEmpty() || + !blacklistedRegions.contains(record.get("location"))) + .collect(Collectors.groupingBy(image -> { + // Retrieve the region + String region = image.optString("location"); + // Retrieve the imageReq + String os = (String) ((JSONObject) image.get("operatingSystem")).get("family"); + os = os.substring(0, 1) + .toUpperCase() + + os.substring(1); + String imageReq = getOsAccordingToCloudProvider(paCloud.getCloudProvider(), + os); + + return new Pair<>(region, + imageReq); + })); + + consolidatedImagesGrouped.entrySet().parallelStream().forEach(entry -> { + String region = entry.getKey().getValue0(); + String imageReq = entry.getKey().getValue1(); + List imageList = entry.getValue(); + + try { + JSONArray nodeCandidates = nodeCandidatesCache.get(Quartet.with(paCloud, region, imageReq, "")); + + imageList.forEach(image -> createAndStoreIaasNodesAndNodeCandidates(nodeCandidates, + paCloud, + image)); + } catch (ExecutionException e) { + LOGGER.error("Could not get node candidates from cache: ", e); } - populateNodeCandidatesFromCache(paCloud, region, imageReq, image); }); + }); repositoryService.flush(); } - private void populateNodeCandidatesFromCache(PACloud paCloud, String region, String imageReq, JSONObject image) { - try { - JSONArray nodeCandidates = nodeCandidatesCache.get(Quartet.with(paCloud, region, imageReq, "")); - nodeCandidates.forEach(nc -> { - JSONObject nodeCandidate = (JSONObject) nc; - createLocation(nodeCandidate, paCloud); - NodeCandidate newNodeCandidate = createNodeCandidate(nodeCandidate, image, paCloud); - repositoryService.saveNodeCandidate(newNodeCandidate); - IaasNode newIaasNode = new IaasNode(newNodeCandidate); - repositoryService.saveIaasNode(newIaasNode); - newNodeCandidate.setNodeId(newIaasNode.getId()); - repositoryService.saveNodeCandidate(newNodeCandidate); - }); - } catch (ExecutionException ee) { - LOGGER.error("Could not get node candidates from cache: ", ee); - } + private void createAndStoreIaasNodesAndNodeCandidates(JSONArray nodeCandidates, PACloud paCloud, JSONObject image) { + nodeCandidates.forEach(nc -> { + JSONObject nodeCandidate = (JSONObject) nc; + createLocation(nodeCandidate, paCloud); + NodeCandidate newNodeCandidate = createNodeCandidate(nodeCandidate, image, paCloud); + repositoryService.saveNodeCandidate(newNodeCandidate); + IaasNode newIaasNode = new IaasNode(newNodeCandidate); + repositoryService.saveIaasNode(newIaasNode); + newNodeCandidate.setNodeId(newIaasNode.getId()); + repositoryService.saveNodeCandidate(newNodeCandidate); + }); } private JSONArray getAllPagedNodeCandidates(PACloud paCloud, String region, String imageReq, String token) { diff --git a/sal-service/src/main/java/org/ow2/proactive/sal/service/service/CloudService.java b/sal-service/src/main/java/org/ow2/proactive/sal/service/service/CloudService.java index 7e06fae..3f8dcb5 100644 --- a/sal-service/src/main/java/org/ow2/proactive/sal/service/service/CloudService.java +++ b/sal-service/src/main/java/org/ow2/proactive/sal/service/service/CloudService.java @@ -97,6 +97,7 @@ public Integer addClouds(String sessionId, List clouds) throws Credentials credentials = new Credentials(); credentials.setUserName(cloud.getCredentials().getUser()); + credentials.setProjectId(cloud.getCredentials().getProjectId()); credentials.setPrivateKey(cloud.getCredentials().getSecret()); credentials.setDomain(cloud.getCredentials().getDomain()); credentials.setSubscriptionId(cloud.getCredentials().getSubscriptionId()); @@ -467,6 +468,9 @@ private Credentials hideCredentials(Credentials creds) { if (creds.getUserName() != null) { newCreds.setUserName(hideString(creds.getUserName(), 5)); } + if (creds.getProjectId() != null) { + newCreds.setProjectId(hideString(creds.getProjectId(), 5)); + } } return newCreds; } diff --git a/sal-service/src/main/java/org/ow2/proactive/sal/service/service/NodeService.java b/sal-service/src/main/java/org/ow2/proactive/sal/service/service/NodeService.java index b1ecd12..a114648 100644 --- a/sal-service/src/main/java/org/ow2/proactive/sal/service/service/NodeService.java +++ b/sal-service/src/main/java/org/ow2/proactive/sal/service/service/NodeService.java @@ -167,6 +167,17 @@ public Deployment addNode(IaasDefinition node, Job job) { return newDeployment; } + private String getZoneFromGceHardwareLocation(String location) { + String[] locationAsArray = location.split("/"); + List locationAsList = Arrays.asList(locationAsArray); + int indexOfZones = locationAsList.indexOf("zones"); + if (indexOfZones == -1) { + return "null"; + } else { + return locationAsArray[indexOfZones + 1]; + } + } + /** * Define a node source in PA server related to a deployment information * @param nodeSourceName A valid and unique node source name @@ -251,6 +262,25 @@ private void defineNSWithDeploymentInfo(String nodeSourceName, PACloud cloud, De variables.put("vmPublicKey", cloud.getSshCredentials().getPublicKey()); variables.put("region", deployment.getNode().getNodeCandidate().getLocation().getName()); break; + case GCE: + filename = File.separator + "Define_NS_GCE.xml"; + variables.put("user", cloud.getCredentials().getUserName()); + variables.put("projectId", cloud.getCredentials().getProjectId()); + variables.put("secret", cloud.getCredentials().getPrivateKey()); + variables.put("vmUsername", cloud.getSshCredentials().getUsername()); + variables.put("vmPublicKey", cloud.getSshCredentials().getPublicKey()); + variables.put("vmPrivateKey", cloud.getSshCredentials().getPrivateKey()); + variables.put("image", deployment.getNode().getNodeCandidate().getImage().getName()); + // To avoid: "Machine type specified xxx is in a different scope than the instance" + variables.put("region", + getZoneFromGceHardwareLocation(deployment.getNode() + .getNodeCandidate() + .getHardware() + .getId())); + variables.put("machineType", deployment.getNode().getNodeCandidate().getHardware().getProviderId()); + variables.put("ram", deployment.getNode().getNodeCandidate().getHardware().getRam().toString()); + variables.put("cores", deployment.getNode().getNodeCandidate().getHardware().getCores().toString()); + break; default: throw new IllegalArgumentException("Unhandled cloud provider: " + cloud.getCloudProvider()); } diff --git a/sal-service/src/main/java/org/ow2/proactive/sal/service/service/TaskBuilder.java b/sal-service/src/main/java/org/ow2/proactive/sal/service/service/TaskBuilder.java index bc5f264..b9174ab 100644 --- a/sal-service/src/main/java/org/ow2/proactive/sal/service/service/TaskBuilder.java +++ b/sal-service/src/main/java/org/ow2/proactive/sal/service/service/TaskBuilder.java @@ -343,53 +343,19 @@ private Map createVariablesMapForAcquiringIAASNode(Task ta return (variablesMap); } - private ScriptTask createInfraIAASTaskForAWS(Task task, Deployment deployment, String taskNameSuffix, - String nodeToken) { - LOGGER.debug("Acquiring node AWS script file: " + - getClass().getResource(File.separator + ACQUIRE_NODE_SCRIPT).toString()); - ScriptTask deployNodeTask = PAFactory.createGroovyScriptTaskFromFile("acquireAWSNode_" + task.getName() + - taskNameSuffix, ACQUIRE_NODE_SCRIPT); - - deployNodeTask.setPreScript(PAFactory.createSimpleScriptFromFIle(PRE_ACQUIRE_NODE_SCRIPT, "groovy")); - - Map variablesMap = createVariablesMapForAcquiringIAASNode(task, deployment, nodeToken); - LOGGER.debug("Variables to be added to the task acquiring AWS IAAS node: " + variablesMap.toString()); - deployNodeTask.setVariables(variablesMap); - - addLocalDefaultNSRegexSelectionScript(deployNodeTask); - - return deployNodeTask; - } - - private ScriptTask createInfraIAASTaskForOS(Task task, Deployment deployment, String taskNameSuffix, - String nodeToken) { - LOGGER.debug("Acquiring node OS script file: " + + private ScriptTask createInfraIAASTaskForCloudProvider(Task task, Deployment deployment, String taskNameSuffix, + String nodeToken, String cloudProvider) { + LOGGER.debug("Acquiring node " + cloudProvider + " script file: " + getClass().getResource(File.separator + ACQUIRE_NODE_SCRIPT).toString()); - ScriptTask deployNodeTask = PAFactory.createGroovyScriptTaskFromFile("acquireOSNode_" + task.getName() + - taskNameSuffix, ACQUIRE_NODE_SCRIPT); - - deployNodeTask.setPreScript(PAFactory.createSimpleScriptFromFIle(PRE_ACQUIRE_NODE_SCRIPT, "groovy")); - - Map variablesMap = createVariablesMapForAcquiringIAASNode(task, deployment, nodeToken); - LOGGER.debug("Variables to be added to the task acquiring OS IAAS node: " + variablesMap.toString()); - deployNodeTask.setVariables(variablesMap); - - addLocalDefaultNSRegexSelectionScript(deployNodeTask); - - return deployNodeTask; - } - - private ScriptTask createInfraIAASTaskForAzure(Task task, Deployment deployment, String taskNameSuffix, - String nodeToken) { - LOGGER.debug("Acquiring node Azure script file: " + - getClass().getResource(File.separator + ACQUIRE_NODE_SCRIPT).toString()); - ScriptTask deployNodeTask = PAFactory.createGroovyScriptTaskFromFile("acquireAzureNode_" + task.getName() + - taskNameSuffix, ACQUIRE_NODE_SCRIPT); + ScriptTask deployNodeTask = PAFactory.createGroovyScriptTaskFromFile("acquire" + cloudProvider + "Node_" + + task.getName() + taskNameSuffix, + ACQUIRE_NODE_SCRIPT); deployNodeTask.setPreScript(PAFactory.createSimpleScriptFromFIle(PRE_ACQUIRE_NODE_SCRIPT, "groovy")); Map variablesMap = createVariablesMapForAcquiringIAASNode(task, deployment, nodeToken); - LOGGER.debug("Variables to be added to the task acquiring Azure IAAS node: " + variablesMap.toString()); + LOGGER.debug("Variables to be added to the task acquiring " + cloudProvider + " IAAS node: " + + variablesMap.toString()); deployNodeTask.setVariables(variablesMap); addLocalDefaultNSRegexSelectionScript(deployNodeTask); @@ -400,11 +366,13 @@ private ScriptTask createInfraIAASTaskForAzure(Task task, Deployment deployment, private ScriptTask createInfraIAASTask(Task task, Deployment deployment, String taskNameSuffix, String nodeToken) { switch (deployment.getPaCloud().getCloudProvider()) { case AWS_EC2: - return createInfraIAASTaskForAWS(task, deployment, taskNameSuffix, nodeToken); + return createInfraIAASTaskForCloudProvider(task, deployment, taskNameSuffix, nodeToken, "AWS"); case OPENSTACK: - return createInfraIAASTaskForOS(task, deployment, taskNameSuffix, nodeToken); + return createInfraIAASTaskForCloudProvider(task, deployment, taskNameSuffix, nodeToken, "OS"); case AZURE: - return createInfraIAASTaskForAzure(task, deployment, taskNameSuffix, nodeToken); + return createInfraIAASTaskForCloudProvider(task, deployment, taskNameSuffix, nodeToken, "Azure"); + case GCE: + return createInfraIAASTaskForCloudProvider(task, deployment, taskNameSuffix, nodeToken, "GCE"); default: return new ScriptTask(); } diff --git a/sal-service/src/main/java/org/ow2/proactive/sal/service/service/infrastructure/PAConnectorIaasGateway.java b/sal-service/src/main/java/org/ow2/proactive/sal/service/service/infrastructure/PAConnectorIaasGateway.java index 4ef1016..98a4c01 100644 --- a/sal-service/src/main/java/org/ow2/proactive/sal/service/service/infrastructure/PAConnectorIaasGateway.java +++ b/sal-service/src/main/java/org/ow2/proactive/sal/service/service/infrastructure/PAConnectorIaasGateway.java @@ -152,6 +152,13 @@ public void defineInfrastructure(String infrastructureName, PACloud cloud, Strin cloud.getCredentials().getDomain() + "\", \"subscriptionId\": \"" + cloud.getCredentials().getSubscriptionId() + "\"}}"; break; + case GCE: + jsonOutputString = "{\"id\": \"" + infrastructureName + "\"," + "\"type\": \"" + + cloud.getCloudProvider() + "\"," + "\"credentials\": {\"username\": \"" + + cloud.getCredentials().getUserName() + "\", \"projectId\": \"" + + cloud.getCredentials().getProjectId() + "\", \"password\": \"" + + cloud.getCredentials().getPrivateKey() + "\"}, \"region\": \"" + region + "\"}"; + break; default: throw new IllegalArgumentException("The infrastructure " + infrastructureName + " is not handled yet."); } diff --git a/sal-service/src/main/resources/Define_NS_GCE.xml b/sal-service/src/main/resources/Define_NS_GCE.xml new file mode 100644 index 0000000..89d20d2 --- /dev/null +++ b/sal-service/src/main/resources/Define_NS_GCE.xml @@ -0,0 +1,127 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 142.125 + + + 351.5 + + + + + + + + + + + + + + + + ]]> + + + \ No newline at end of file