GCE integration

sal-common/src/main/java/org/ow2/proactive/sal/model/CloudProviderType.java

@@ -16,7 +16,7 @@ public enum CloudProviderType {
 
     AWS_EC2("aws-ec2"),
     AZURE("azure"),
-    GCE("gce"),
+    GCE("google-compute-engine"),
     OPENSTACK("openstack"), //openstack-nova
 
     EDGE("EDGE"),

sal-common/src/main/java/org/ow2/proactive/sal/model/Credential.java

@@ -28,6 +28,8 @@ public class Credential implements Serializable {
     // JSON field constants
     public static final String JSON_USER = "user";
 
+    public static final String JSON_PROJECT_ID = "projectId";
+
     public static final String JSON_SECRET = "secret";
 
     public static final String JSON_DOMAIN = "domain";
@@ -37,6 +39,9 @@ public class Credential implements Serializable {
     @JsonProperty(JSON_USER)
     private String user = null;
 
+    @JsonProperty(JSON_PROJECT_ID)
+    private String projectId = null;
+
     @JsonProperty(JSON_SECRET)
     private String secret = null;
 

sal-common/src/main/java/org/ow2/proactive/sal/model/Credentials.java

@@ -31,6 +31,8 @@ public class Credentials implements Serializable {
 
     public static final String JSON_USER_NAME = "userName";
 
+    public static final String JSON_PROJECT_ID = "projectId";
+
     public static final String JSON_PASSWORD = "password";
 
     public static final String JSON_PRIVATE_KEY = "privateKey";
@@ -51,6 +53,10 @@ public class Credentials implements Serializable {
     @JsonProperty(JSON_USER_NAME)
     private String userName;
 
+    @Column(name = "PROJECT_ID")
+    @JsonProperty(JSON_PROJECT_ID)
+    private String projectId;
+
     @Column(name = "PASSWORD")
     @JsonProperty(JSON_PASSWORD)
     private String password;

sal-common/src/main/java/org/ow2/proactive/sal/model/EmsDeploymentRequest.java

@@ -59,7 +59,7 @@ public enum TargetProvider {
         // Azure VM
         AZUREVM("azure"),
         // Google CLoud Engine
-        GCE("gce"),
+        GCE("google-compute-engine"),
         // OpenStack NOVA
         OPENSTACKNOVA("openstack"),
         // BYON, to be used for on-premise baremetal

sal-service/src/main/java/org/ow2/proactive/sal/service/nc/NodeCandidateUtils.java

@@ -18,6 +18,7 @@
 import javax.annotation.PostConstruct;
 
 import org.apache.commons.lang3.StringUtils;
+import org.javatuples.Pair;
 import org.javatuples.Quartet;
 import org.json.JSONArray;
 import org.json.JSONObject;
@@ -363,6 +364,22 @@ private static JSONObject convertObjectToJson(Object object) {
         return myJson;
     }
 
+    private String getOsAccordingToCloudProvider(CloudProviderType cloudProviderType, String os)
+            throws IllegalArgumentException {
+        switch (cloudProviderType) {
+            case AWS_EC2:
+                return "Linux";
+            case OPENSTACK:
+                return os;
+            case AZURE:
+                return os;
+            case GCE:
+                return os;
+            default:
+                throw new IllegalArgumentException("The infrastructure " + cloudProviderType + " is not handled yet.");
+        }
+    }
+
     public void saveNodeCandidates(List<String> newCloudIds) {
         newCloudIds.forEach(newCloudId -> {
             PACloud paCloud = repositoryService.getPACloud(newCloudId);
@@ -378,65 +395,60 @@ public void saveNodeCandidates(List<String> newCloudIds) {
                 return;
             }
             LOGGER.info("Returned images: {}", images);
-            List<JSONObject> consolidatedImages = images.toList()
-                                                        .parallelStream()
-                                                        .map(NodeCandidateUtils::convertObjectToJson)
-                                                        .filter(record -> !blacklistedRegions.contains(record.get("location")))
-                                                        .collect(Collectors.toList());
-            LOGGER.info("Consolidated images: {}", consolidatedImages);
-
-            //TODO: (Optimization) An images per region map structure <region,[image1,image2]> could be the best here.
-            // It can reduce the getNodeCandidates calls to PA.
-            List<String> entries = new LinkedList<>();
-            List<String> openstackOsList = Arrays.asList("Ubuntu", "Fedora", "Centos", "Debian");
-            consolidatedImages.forEach(image -> {
-                String region = image.optString("location");
-                String imageReq;
-                String os = (String) ((JSONObject) image.get("operatingSystem")).get("family");
-                os = os.substring(0, 1).toUpperCase() + os.substring(1);
-                String pair = os + ":" + region;
-
-                switch (paCloud.getCloudProvider()) {
-                    case AWS_EC2:
-                        imageReq = "Linux";
-                        break;
-                    case OPENSTACK:
-                        imageReq = os;
-                        break;
-                    case AZURE:
-                        imageReq = os;
-                        break;
-                    default:
-                        throw new IllegalArgumentException("The infrastructure " + paCloud.getCloudType() +
-                                                           " is not handled yet.");
-                }
-
-                if (paCloud.getCloudProvider() == OPENSTACK) {
-                    entries.add(pair);
+            Map<Pair<String, String>, List<JSONObject>> consolidatedImagesGrouped = images.toList()
+                                                                                          .parallelStream()
+                                                                                          .map(NodeCandidateUtils::convertObjectToJson)
+                                                                                          .filter(record -> record.get("location")
+                                                                                                                  .toString()
+                                                                                                                  .isEmpty() ||
+                                                                                                            !blacklistedRegions.contains(record.get("location")))
+                                                                                          .collect(Collectors.groupingBy(image -> {
+                                                                                              // Retrieve the region
+                                                                                              String region = image.optString("location");
+                                                                                              // Retrieve the imageReq
+                                                                                              String os = (String) ((JSONObject) image.get("operatingSystem")).get("family");
+                                                                                              os = os.substring(0, 1)
+                                                                                                     .toUpperCase() +
+                                                                                                   os.substring(1);
+                                                                                              String imageReq = getOsAccordingToCloudProvider(paCloud.getCloudProvider(),
+                                                                                                                                              os);
+
+                                                                                              return new Pair<>(region,
+                                                                                                                imageReq);
+                                                                                          }));
+
+            consolidatedImagesGrouped.entrySet().parallelStream().forEach(entry -> {
+                String region = entry.getKey().getValue0();
+                String imageReq = entry.getKey().getValue1();
+                List<JSONObject> imageList = entry.getValue();
+
+                try {
+                    JSONArray nodeCandidates = nodeCandidatesCache.get(Quartet.with(paCloud, region, imageReq, ""));
+
+                    imageList.forEach(image -> createAndStoreIaasNodesAndNodeCandidates(nodeCandidates,
+                                                                                        paCloud,
+                                                                                        image));
+                } catch (ExecutionException e) {
+                    LOGGER.error("Could not get node candidates from cache: ", e);
                 }
-                populateNodeCandidatesFromCache(paCloud, region, imageReq, image);
             });
+
         });
 
         repositoryService.flush();
     }
 
-    private void populateNodeCandidatesFromCache(PACloud paCloud, String region, String imageReq, JSONObject image) {
-        try {
-            JSONArray nodeCandidates = nodeCandidatesCache.get(Quartet.with(paCloud, region, imageReq, ""));
-            nodeCandidates.forEach(nc -> {
-                JSONObject nodeCandidate = (JSONObject) nc;
-                createLocation(nodeCandidate, paCloud);
-                NodeCandidate newNodeCandidate = createNodeCandidate(nodeCandidate, image, paCloud);
-                repositoryService.saveNodeCandidate(newNodeCandidate);
-                IaasNode newIaasNode = new IaasNode(newNodeCandidate);
-                repositoryService.saveIaasNode(newIaasNode);
-                newNodeCandidate.setNodeId(newIaasNode.getId());
-                repositoryService.saveNodeCandidate(newNodeCandidate);
-            });
-        } catch (ExecutionException ee) {
-            LOGGER.error("Could not get node candidates from cache: ", ee);
-        }
+    private void createAndStoreIaasNodesAndNodeCandidates(JSONArray nodeCandidates, PACloud paCloud, JSONObject image) {
+        nodeCandidates.forEach(nc -> {
+            JSONObject nodeCandidate = (JSONObject) nc;
+            createLocation(nodeCandidate, paCloud);
+            NodeCandidate newNodeCandidate = createNodeCandidate(nodeCandidate, image, paCloud);
+            repositoryService.saveNodeCandidate(newNodeCandidate);
+            IaasNode newIaasNode = new IaasNode(newNodeCandidate);
+            repositoryService.saveIaasNode(newIaasNode);
+            newNodeCandidate.setNodeId(newIaasNode.getId());
+            repositoryService.saveNodeCandidate(newNodeCandidate);
+        });
     }
 
     private JSONArray getAllPagedNodeCandidates(PACloud paCloud, String region, String imageReq, String token) {

sal-service/src/main/java/org/ow2/proactive/sal/service/service/CloudService.java

@@ -97,6 +97,7 @@ public Integer addClouds(String sessionId, List<CloudDefinition> clouds) throws
 
             Credentials credentials = new Credentials();
             credentials.setUserName(cloud.getCredentials().getUser());
+            credentials.setProjectId(cloud.getCredentials().getProjectId());
             credentials.setPrivateKey(cloud.getCredentials().getSecret());
             credentials.setDomain(cloud.getCredentials().getDomain());
             credentials.setSubscriptionId(cloud.getCredentials().getSubscriptionId());
@@ -467,6 +468,9 @@ private Credentials hideCredentials(Credentials creds) {
             if (creds.getUserName() != null) {
                 newCreds.setUserName(hideString(creds.getUserName(), 5));
             }
+            if (creds.getProjectId() != null) {
+                newCreds.setProjectId(hideString(creds.getProjectId(), 5));
+            }
         }
         return newCreds;
     }

sal-service/src/main/java/org/ow2/proactive/sal/service/service/NodeService.java

@@ -167,6 +167,17 @@ public Deployment addNode(IaasDefinition node, Job job) {
         return newDeployment;
     }
 
+    private String getZoneFromGceHardwareLocation(String location) {
+        String[] locationAsArray = location.split("/");
+        List<String> locationAsList = Arrays.asList(locationAsArray);
+        int indexOfZones = locationAsList.indexOf("zones");
+        if (indexOfZones == -1) {
+            return "null";
+        } else {
+            return locationAsArray[indexOfZones + 1];
+        }
+    }
+
     /**
      * Define a node source in PA server related to a deployment information
      * @param nodeSourceName A valid and unique node source name
@@ -251,6 +262,25 @@ private void defineNSWithDeploymentInfo(String nodeSourceName, PACloud cloud, De
                     variables.put("vmPublicKey", cloud.getSshCredentials().getPublicKey());
                     variables.put("region", deployment.getNode().getNodeCandidate().getLocation().getName());
                     break;
+                case GCE:
+                    filename = File.separator + "Define_NS_GCE.xml";
+                    variables.put("user", cloud.getCredentials().getUserName());
+                    variables.put("projectId", cloud.getCredentials().getProjectId());
+                    variables.put("secret", cloud.getCredentials().getPrivateKey());
+                    variables.put("vmUsername", cloud.getSshCredentials().getUsername());
+                    variables.put("vmPublicKey", cloud.getSshCredentials().getPublicKey());
+                    variables.put("vmPrivateKey", cloud.getSshCredentials().getPrivateKey());
+                    variables.put("image", deployment.getNode().getNodeCandidate().getImage().getName());
+                    // To avoid: "Machine type specified xxx is in a different scope than the instance"
+                    variables.put("region",
+                                  getZoneFromGceHardwareLocation(deployment.getNode()
+                                                                           .getNodeCandidate()
+                                                                           .getHardware()
+                                                                           .getId()));
+                    variables.put("machineType", deployment.getNode().getNodeCandidate().getHardware().getProviderId());
+                    variables.put("ram", deployment.getNode().getNodeCandidate().getHardware().getRam().toString());
+                    variables.put("cores", deployment.getNode().getNodeCandidate().getHardware().getCores().toString());
+                    break;
                 default:
                     throw new IllegalArgumentException("Unhandled cloud provider: " + cloud.getCloudProvider());
             }

sal-service/src/main/java/org/ow2/proactive/sal/service/service/TaskBuilder.java

@@ -343,53 +343,19 @@ private Map<String, TaskVariable> createVariablesMapForAcquiringIAASNode(Task ta
         return (variablesMap);
     }
 
-    private ScriptTask createInfraIAASTaskForAWS(Task task, Deployment deployment, String taskNameSuffix,
-            String nodeToken) {
-        LOGGER.debug("Acquiring node AWS script file: " +
-                     getClass().getResource(File.separator + ACQUIRE_NODE_SCRIPT).toString());
-        ScriptTask deployNodeTask = PAFactory.createGroovyScriptTaskFromFile("acquireAWSNode_" + task.getName() +
-                                                                             taskNameSuffix, ACQUIRE_NODE_SCRIPT);
-
-        deployNodeTask.setPreScript(PAFactory.createSimpleScriptFromFIle(PRE_ACQUIRE_NODE_SCRIPT, "groovy"));
-
-        Map<String, TaskVariable> variablesMap = createVariablesMapForAcquiringIAASNode(task, deployment, nodeToken);
-        LOGGER.debug("Variables to be added to the task acquiring AWS IAAS node: " + variablesMap.toString());
-        deployNodeTask.setVariables(variablesMap);
-
-        addLocalDefaultNSRegexSelectionScript(deployNodeTask);
-
-        return deployNodeTask;
-    }
-
-    private ScriptTask createInfraIAASTaskForOS(Task task, Deployment deployment, String taskNameSuffix,
-            String nodeToken) {
-        LOGGER.debug("Acquiring node OS script file: " +
+    private ScriptTask createInfraIAASTaskForCloudProvider(Task task, Deployment deployment, String taskNameSuffix,
+            String nodeToken, String cloudProvider) {
+        LOGGER.debug("Acquiring node " + cloudProvider + " script file: " +
                      getClass().getResource(File.separator + ACQUIRE_NODE_SCRIPT).toString());
-        ScriptTask deployNodeTask = PAFactory.createGroovyScriptTaskFromFile("acquireOSNode_" + task.getName() +
-                                                                             taskNameSuffix, ACQUIRE_NODE_SCRIPT);
-
-        deployNodeTask.setPreScript(PAFactory.createSimpleScriptFromFIle(PRE_ACQUIRE_NODE_SCRIPT, "groovy"));
-
-        Map<String, TaskVariable> variablesMap = createVariablesMapForAcquiringIAASNode(task, deployment, nodeToken);
-        LOGGER.debug("Variables to be added to the task acquiring OS IAAS node: " + variablesMap.toString());
-        deployNodeTask.setVariables(variablesMap);
-
-        addLocalDefaultNSRegexSelectionScript(deployNodeTask);
-
-        return deployNodeTask;
-    }
-
-    private ScriptTask createInfraIAASTaskForAzure(Task task, Deployment deployment, String taskNameSuffix,
-            String nodeToken) {
-        LOGGER.debug("Acquiring node Azure script file: " +
-                     getClass().getResource(File.separator + ACQUIRE_NODE_SCRIPT).toString());
-        ScriptTask deployNodeTask = PAFactory.createGroovyScriptTaskFromFile("acquireAzureNode_" + task.getName() +
-                                                                             taskNameSuffix, ACQUIRE_NODE_SCRIPT);
+        ScriptTask deployNodeTask = PAFactory.createGroovyScriptTaskFromFile("acquire" + cloudProvider + "Node_" +
+                                                                             task.getName() + taskNameSuffix,
+                                                                             ACQUIRE_NODE_SCRIPT);
 
         deployNodeTask.setPreScript(PAFactory.createSimpleScriptFromFIle(PRE_ACQUIRE_NODE_SCRIPT, "groovy"));
 
         Map<String, TaskVariable> variablesMap = createVariablesMapForAcquiringIAASNode(task, deployment, nodeToken);
-        LOGGER.debug("Variables to be added to the task acquiring Azure IAAS node: " + variablesMap.toString());
+        LOGGER.debug("Variables to be added to the task acquiring " + cloudProvider + " IAAS node: " +
+                     variablesMap.toString());
         deployNodeTask.setVariables(variablesMap);
 
         addLocalDefaultNSRegexSelectionScript(deployNodeTask);
@@ -400,11 +366,13 @@ private ScriptTask createInfraIAASTaskForAzure(Task task, Deployment deployment,
     private ScriptTask createInfraIAASTask(Task task, Deployment deployment, String taskNameSuffix, String nodeToken) {
         switch (deployment.getPaCloud().getCloudProvider()) {
             case AWS_EC2:
-                return createInfraIAASTaskForAWS(task, deployment, taskNameSuffix, nodeToken);
+                return createInfraIAASTaskForCloudProvider(task, deployment, taskNameSuffix, nodeToken, "AWS");
             case OPENSTACK:
-                return createInfraIAASTaskForOS(task, deployment, taskNameSuffix, nodeToken);
+                return createInfraIAASTaskForCloudProvider(task, deployment, taskNameSuffix, nodeToken, "OS");
             case AZURE:
-                return createInfraIAASTaskForAzure(task, deployment, taskNameSuffix, nodeToken);
+                return createInfraIAASTaskForCloudProvider(task, deployment, taskNameSuffix, nodeToken, "Azure");
+            case GCE:
+                return createInfraIAASTaskForCloudProvider(task, deployment, taskNameSuffix, nodeToken, "GCE");
             default:
                 return new ScriptTask();
         }

sal-service/src/main/java/org/ow2/proactive/sal/service/service/infrastructure/PAConnectorIaasGateway.java

@@ -152,6 +152,13 @@ public void defineInfrastructure(String infrastructureName, PACloud cloud, Strin
                                    cloud.getCredentials().getDomain() + "\", \"subscriptionId\": \"" +
                                    cloud.getCredentials().getSubscriptionId() + "\"}}";
                 break;
+            case GCE:
+                jsonOutputString = "{\"id\": \"" + infrastructureName + "\"," + "\"type\": \"" +
+                                   cloud.getCloudProvider() + "\"," + "\"credentials\": {\"username\": \"" +
+                                   cloud.getCredentials().getUserName() + "\", \"projectId\": \"" +
+                                   cloud.getCredentials().getProjectId() + "\", \"password\": \"" +
+                                   cloud.getCredentials().getPrivateKey() + "\"}, \"region\": \"" + region + "\"}";
+                break;
             default:
                 throw new IllegalArgumentException("The infrastructure " + infrastructureName + " is not handled yet.");
         }