fix cloud storm DB and test credential service

drip-commons/src/main/java/nl/uva/sne/drip/commons/utils/ToscaHelper.java

@@ -15,6 +15,7 @@
  */
 package nl.uva.sne.drip.commons.utils;
 
+import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.SerializationFeature;
@@ -41,9 +42,9 @@ import org.springframework.beans.factory.annotation.Autowired;
  * @author S. Koulouzis
  */
 public class ToscaHelper {
-    
+
     private DefaultApi api;
-    
+
     private ObjectMapper objectMapper;
     public static final String VM_CAPABILITY = "tosca.capabilities.ARTICONF.VM";
     private static final String VM_TYPE = "tosca.nodes.ARTICONF.VM.Compute";
@@ -52,7 +53,7 @@ public class ToscaHelper {
     private static final String VM_OS = "os";
     private static final String VM_TOPOLOGY = "tosca.nodes.ARTICONF.VM.topology";
     private Integer id;
-    
+
     @Autowired
     public ToscaHelper(String sureToscaBasePath) {
         init(sureToscaBasePath);
@@ -64,14 +65,16 @@ public class ToscaHelper {
     public Integer getId() {
         return id;
     }
-    
+
     private void init(String sureToscaBasePath) {
         Configuration.getDefaultApiClient().setBasePath(sureToscaBasePath);
         api = new DefaultApi(Configuration.getDefaultApiClient());
         this.objectMapper = new ObjectMapper(new YAMLFactory().disable(YAMLGenerator.Feature.WRITE_DOC_START_MARKER));
         objectMapper.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS);
+        objectMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL);
+
     }
-    
+
     public void uploadToscaTemplate(ToscaTemplate toscaTemplate) throws JsonProcessingException, IOException, ApiException {
         String ymlStr = objectMapper.writeValueAsString(toscaTemplate);
         File toscaTemplateFile = File.createTempFile("temp-toscaTemplate", ".yml");
@@ -79,7 +82,7 @@ public class ToscaHelper {
         String resp = api.uploadToscaTemplate(toscaTemplateFile);
         id = Integer.valueOf(resp);
     }
-    
+
     public List<Map<String, Object>> getProvisionInterfaceDefinitions(List<String> toscaInterfaceTypes) throws ApiException {
         List<Map<String, Object>> interfaceDefinitions = new ArrayList<>();
         for (String type : toscaInterfaceTypes) {
@@ -89,12 +92,12 @@ public class ToscaHelper {
         }
         return interfaceDefinitions;
     }
-    
+
     public List<NodeTemplateMap> getVMTopologyTemplates() throws ApiException {
         List<NodeTemplateMap> vmTopologyTemplates = api.getNodeTemplates(String.valueOf(id), "tosca.nodes.ARTICONF.VM.topology", null, null, null, null, null, null, null);
         return vmTopologyTemplates;
     }
-    
+
     public List<NodeTemplateMap> getTemplateVMsForVMTopology(NodeTemplateMap nodeTemplateMap) throws ApiException {
         NodeTemplate nodeTemplate = nodeTemplateMap.getNodeTemplate();
         List<Map<String, Object>> requirements = nodeTemplate.getRequirements();
@@ -109,9 +112,9 @@ public class ToscaHelper {
             }
         }
         return vms;
-        
+
     }
-    
+
     public Double getVMNumOfCores(NodeTemplateMap vmMap) throws Exception {
         NodeTemplate vm = vmMap.getNodeTemplate();
         if (vm.getType().equals(VM_TYPE)) {
@@ -120,7 +123,7 @@ public class ToscaHelper {
             throw new Exception("NodeTemplate is not of type: " + VM_TYPE + " it is of type: " + vm.getType());
         }
     }
-    
+
     public Double getVMNMemSize(NodeTemplateMap vmMap) throws Exception {
         NodeTemplate vm = vmMap.getNodeTemplate();
         if (vm.getType().equals(VM_TYPE)) {
@@ -133,9 +136,9 @@ public class ToscaHelper {
         } else {
             throw new Exception("NodeTemplate is not of type: " + VM_TYPE + " it is of type: " + vm.getType());
         }
-        
+
     }
-    
+
     public String getVMNOS(NodeTemplateMap vmMap) throws Exception {
         NodeTemplate vm = vmMap.getNodeTemplate();
         if (vm.getType().equals(VM_TYPE)) {
@@ -144,7 +147,7 @@ public class ToscaHelper {
             throw new Exception("NodeTemplate is not of type: " + VM_TYPE + " it is of type: " + vm.getType());
         }
     }
-    
+
     private Double convertToGB(Integer value, String memUnit) {
         switch (memUnit) {
             case "GB":
@@ -157,7 +160,7 @@ public class ToscaHelper {
                 return null;
         }
     }
-    
+
     public String getTopologyDomain(NodeTemplateMap nodeTemplateMap) throws Exception {
         NodeTemplate nodeTemplate = nodeTemplateMap.getNodeTemplate();
         if (nodeTemplate.getType().equals(VM_TOPOLOGY)) {
@@ -166,7 +169,7 @@ public class ToscaHelper {
             throw new Exception("NodeTemplateMap is not of type: " + VM_TOPOLOGY + " it is of type: " + nodeTemplate.getType());
         }
     }
-    
+
     public String getTopologyProvider(NodeTemplateMap nodeTemplateMap) throws Exception {
         NodeTemplate nodeTemplate = nodeTemplateMap.getNodeTemplate();
         if (nodeTemplate.getType().equals(VM_TOPOLOGY)) {
@@ -175,7 +178,7 @@ public class ToscaHelper {
             throw new Exception("NodeTemplate is not of type: " + VM_TOPOLOGY + " it is of type: " + nodeTemplate.getType());
         }
     }
-    
+
     public NodeTemplateMap setCredentialsInVMTopology(NodeTemplateMap vmTopologyMap, Credential credential) throws Exception {
         NodeTemplate vmTopology = vmTopologyMap.getNodeTemplate();
         if (vmTopology.getType().equals(VM_TOPOLOGY)) {
@@ -198,7 +201,7 @@ public class ToscaHelper {
             throw new Exception("NodeTemplate is not of type: " + VM_TOPOLOGY + " it is of type: " + vmTopology.getType());
         }
     }
-    
+
     public Credential getCredentialsFromVMTopology(NodeTemplateMap vmTopologyMap) throws Exception {
         NodeTemplate vmTopology = vmTopologyMap.getNodeTemplate();
         if (vmTopology.getType().equals(VM_TOPOLOGY)) {
@@ -206,12 +209,12 @@ public class ToscaHelper {
             String ymlStr = Converter.map2YmlString((Map<String, Object>) att.get("credential"));
             Credential toscaCredential = objectMapper.readValue(ymlStr, Credential.class);
             return toscaCredential;
-            
+
         } else {
             throw new Exception("NodeTemplate is not of type: " + VM_TOPOLOGY + " it is of type: " + vmTopology.getType());
         }
     }
-    
+
     public ToscaTemplate setVMTopologyInToscaTemplate(ToscaTemplate toscaTemplate, NodeTemplateMap vmTopologyMap) {
         Map<String, NodeTemplate> nodes = toscaTemplate.getTopologyTemplate().getNodeTemplates();
         nodes.put(vmTopologyMap.getName(), vmTopologyMap.getNodeTemplate());
@@ -230,7 +233,7 @@ public class ToscaHelper {
         }
         return null;
     }
-    
+
     private Map<String, Object> getProvisionInterfaceInstanceDefaultValues(Map<String, Object> definition, String operation) throws ApiException {
         String type = definition.keySet().iterator().next();
         String[] typeArray = type.split("\\.");
@@ -250,7 +253,7 @@ public class ToscaHelper {
     public Map<String, Object> getProvisionerInterfaceFromVMTopology(NodeTemplateMap vmTopologyMap) {
         return (Map<String, Object>) vmTopologyMap.getNodeTemplate().getInterfaces().get("CloudsStorm");
     }
-    
+
     public NodeTemplateMap setProvisionerInterfaceInVMTopology(NodeTemplateMap vmTopologyMap, Map<String, Object> provisionerInterface) {
         throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates.
     }

drip-commons/src/test/java/nl/uva/sne/drip/commons/utils/ToscaHelperTest.java

@@ -15,6 +15,7 @@
  */
 package nl.uva.sne.drip.commons.utils;
 
+import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
@@ -70,6 +71,8 @@ public class ToscaHelperTest {
         byte[] bytes = Files.readAllBytes(Paths.get(testUpdatedApplicationExampleToscaFilePath));
         String ymlStr = new String(bytes, "UTF-8");
         objectMapper = new ObjectMapper(new YAMLFactory().disable(YAMLGenerator.Feature.WRITE_DOC_START_MARKER));
+        objectMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL);
+
         toscaTemplate = objectMapper.readValue(ymlStr, ToscaTemplate.class);
         String serviceBasePath = prop.getProperty("sure-tosca.base.path");
         serviceUp = isServiceUp(serviceBasePath);
@@ -228,7 +231,6 @@ public class ToscaHelperTest {
             String operation = "provision";
 
 //            for (NodeTemplateMap vmTopologyMap : vmTopologies) {
-
 //                Map<String, Object> provisionInterface = instance.getProvisionInterface(provisioner, operation);
 //                List<String> objects = new ArrayList<>();
 //                objects.add("subtopology");
@@ -249,7 +251,6 @@ public class ToscaHelperTest {
 //            topology_1 = toscaTemplateWithCredentials.getTopologyTemplate().getNodeTemplates().get("topology_1");
 //
 //            topology = toscaTemplateWithCredentials.getTopologyTemplate().getNodeTemplates().get("topology");
-
             instance.uploadToscaTemplate(toscaTemplate);
         }
     }

drip-manager/src/main/java/nl/uva/sne/drip/service/ToscaTemplateService.java

@@ -5,6 +5,7 @@
  */
 package nl.uva.sne.drip.service;
 
+import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.SerializationFeature;
@@ -33,6 +34,7 @@ public class ToscaTemplateService {
     public ToscaTemplateService() {
         this.objectMapper = new ObjectMapper(new YAMLFactory().disable(Feature.WRITE_DOC_START_MARKER));
         objectMapper.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS);
+        objectMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL);
     }
 
     @Autowired

drip-manager/src/test/java/nl/uva/sne/drip/service/ServiceTests.java

@@ -28,12 +28,18 @@ import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;
 import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.Base64;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.NoSuchElementException;
 import java.util.logging.Level;
 import java.util.logging.Logger;
+import javax.xml.bind.DatatypeConverter;
 import nl.uva.sne.drip.Swagger2SpringBoot;
 import nl.uva.sne.drip.configuration.MongoConfig;
 import nl.uva.sne.drip.model.tosca.Credential;
@@ -62,6 +68,7 @@ public class ServiceTests {
     private static final String testApplicationExampleToscaFilePath = ".." + File.separator + "TOSCA" + File.separator + "application_example_updated.yaml";
     private static final String testUpdatedApplicationExampleToscaFilePath = ".." + File.separator + "TOSCA" + File.separator + "application_example_updated.yaml";
     private static final String testOutputApplicationExampleToscaFilePath = ".." + File.separator + "TOSCA" + File.separator + "application_example_updated.yaml";
+    private static final String testCredentialPath = ".." + File.separator + "fake_credentials" + File.separator + "test-geni.jks";
 
     @Autowired
     CredentialService credentialService;
@@ -264,6 +271,53 @@ public class ServiceTests {
         saveCredential();
     }
 
+    @Test
+    public void testCredentialService() throws IOException, NoSuchAlgorithmException {
+        Logger.getLogger(ServiceTests.class.getName()).log(Level.INFO, "testCredentialService");
+        MessageDigest md = MessageDigest.getInstance("MD5");
+        md.update(Files.readAllBytes(Paths.get(testCredentialPath)));
+        byte[] digest = md.digest();
+        String fileChecksum = DatatypeConverter
+                .printHexBinary(digest).toUpperCase();
+
+        String keyStore = new String(Files.readAllBytes(Paths.get(testCredentialPath)));
+
+        byte[] encodedBytes = Base64.getEncoder().encode(keyStore.getBytes());
+        String keyStoreEncoded = new String(encodedBytes, "UTF-8");
+
+        Credential credential = new Credential();
+        credential.setCloudProviderName("exogeni");
+        Map<String, String> keys = new HashMap<>();
+        keys.put("keystore", keyStoreEncoded);
+        credential.setKeys(keys);
+        credential.setToken("1234");
+        credential.setTokenType("password");
+        credential.setUser("user");
+
+        byte[] decodedBytes = Base64.getDecoder().decode(keys.get("keystore"));
+        md = MessageDigest.getInstance("MD5");
+        md.update(decodedBytes);        
+        digest = md.digest();
+        String credentialChecksum = DatatypeConverter
+                .printHexBinary(digest).toUpperCase();   
+        
+        assertEquals(fileChecksum, credentialChecksum);
+        
+
+        HashMap<Object, Object> att = new HashMap<>();
+        Map<String, Object> toscaCredential = new HashMap<>();
+        toscaCredential.put("protocol", credential.getProtocol());
+        toscaCredential.put("token_type", credential.getTokenType());
+        toscaCredential.put("token", credential.getToken());
+        toscaCredential.put("keys", credential.getKeys());
+        toscaCredential.put("user", credential.getUser());
+        toscaCredential.put("cloud_provider_name", credential.getCloudProviderName());
+        att.put("credential", toscaCredential);
+
+        
+
+    }
+
     public String saveCredential() {
         Logger.getLogger(ServiceTests.class.getName()).log(Level.INFO, "saveCredential");
         Credential document = new Credential();

drip-provisioner/etc/UD/ExoGENI.yml

@@ -51,6 +51,26 @@ DCMetaInfo:
       OS_URL: "http://geni-images.renci.org/images/standard/ubuntu/ub1404-v1.0.4.xml"
       OS_GUID: "9394ca154aa35eb55e604503ae7943ddaecc6ca5"
       DiskSize: 75
+  - OS: "Ubuntu 18.04"
+    CPU: 4
+    MEM: 12
+    VMType: "XOXLarge"
+    Price: null
+    DefaultSSHAccount: "root"
+    extraInfo:
+      OS_URL: "http://geni-images.renci.org/images/standard/ubuntu-comet/ubuntu-18.04/ubuntu-18.04.xml"
+      OS_GUID: "fe66b08bffa14c385ad64edd124350a2da826af9"
+      DiskSize: 75      
+  - OS: "Ubuntu 18.04"
+    CPU: 2
+    MEM: 6
+    VMType: "XOLarge"
+    Price: null
+    DefaultSSHAccount: "root"
+    extraInfo:
+      OS_URL: "http://geni-images.renci.org/images/standard/ubuntu-comet/ubuntu-18.04/ubuntu-18.04.xml"
+      OS_GUID: "fe66b08bffa14c385ad64edd124350a2da826af9"
+      DiskSize: 50        
 - domain: "BBN/GPO (Boston, MA USA) XO Rack"
   endpoint: "bbnvmsite.rdf#bbnvmsite"
   country: USA
@@ -100,6 +120,26 @@ DCMetaInfo:
       OS_URL: "http://geni-images.renci.org/images/standard/ubuntu/ub1404-v1.0.4.xml"
       OS_GUID: "9394ca154aa35eb55e604503ae7943ddaecc6ca5"
       DiskSize: 75
+  - OS: "Ubuntu 18.04"
+    CPU: 4
+    MEM: 12
+    VMType: "XOXLarge"
+    Price: null
+    DefaultSSHAccount: "root"
+    extraInfo:
+      OS_URL: "http://geni-images.renci.org/images/standard/ubuntu-comet/ubuntu-18.04/ubuntu-18.04.xml"
+      OS_GUID: "fe66b08bffa14c385ad64edd124350a2da826af9"
+      DiskSize: 75      
+  - OS: "Ubuntu 18.04"
+    CPU: 2
+    MEM: 6
+    VMType: "XOLarge"
+    Price: null
+    DefaultSSHAccount: "root"
+    extraInfo:
+      OS_URL: "http://geni-images.renci.org/images/standard/ubuntu-comet/ubuntu-18.04/ubuntu-18.04.xml"
+      OS_GUID: "fe66b08bffa14c385ad64edd124350a2da826af9"
+      DiskSize: 50              
 - domain: "NICTA (Sydney, Australia) XO Rack"
   endpoint: "nictavmsite.rdf#nictavmsite"
   country: AUS
@@ -834,24 +874,4 @@ DCMetaInfo:
     extraInfo:
       OS_URL: "http://geni-images.renci.org/images/standard/ubuntu/ub1404-v1.0.4.xml"
       OS_GUID: "9394ca154aa35eb55e604503ae7943ddaecc6ca5"
-      DiskSize: 75
-  - OS: "Ubuntu 18.04"
-    CPU: 4
-    MEM: 12
-    VMType: "XOXLarge"
-    Price: null
-    DefaultSSHAccount: "root"
-    extraInfo:
-      OS_URL: "http://geni-images.renci.org/images/standard/ubuntu-comet/ubuntu-18.04/ubuntu-18.04.xml"
-      OS_GUID: "fe66b08bffa14c385ad64edd124350a2da826af9"
-      DiskSize: 75      
-  - OS: "Ubuntu 18.04"
-    CPU: 2
-    MEM: 6
-    VMType: "XOLarge"
-    Price: null
-    DefaultSSHAccount: "root"
-    extraInfo:
-      OS_URL: "http://geni-images.renci.org/images/standard/ubuntu-comet/ubuntu-18.04/ubuntu-18.04.xml"
-      OS_GUID: "fe66b08bffa14c385ad64edd124350a2da826af9"
-      DiskSize: 50          
+      DiskSize: 75        

drip-provisioner/pom.xml

@@ -63,7 +63,7 @@
         <dependency>
             <groupId>nl.uva.sne.zh</groupId>
             <artifactId>CloudsStorm</artifactId>
-            <version>1.0</version>
+            <version>b.1.0</version>
         </dependency>
     </dependencies>
    

drip-provisioner/src/main/java/nl/uva/sne/drip/provisioner/CloudStormDAO.java

@@ -5,6 +5,7 @@
  */
 package nl.uva.sne.drip.provisioner;
 
+import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.SerializationFeature;
 import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
@@ -34,6 +35,7 @@ class CloudStormDAO {
         this.cloudStormDBPath = cloudStormDBPath;
         this.objectMapper = new ObjectMapper(new YAMLFactory().disable(YAMLGenerator.Feature.WRITE_DOC_START_MARKER));
         objectMapper.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS);
+        objectMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL);
     }
 
     List<VMMetaInfo> findVmMetaInfoByProvider(CloudDB.CloudProviderEnum provider) throws IOException {

drip-provisioner/src/main/java/nl/uva/sne/drip/provisioner/CloudStormService.java

@@ -5,6 +5,7 @@
  */
 package nl.uva.sne.drip.provisioner;
 
+import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.SerializationFeature;
@@ -69,6 +70,7 @@ class CloudStormService {
         this.helper.uploadToscaTemplate(toscaTemplate);
         this.objectMapper = new ObjectMapper(new YAMLFactory().disable(YAMLGenerator.Feature.WRITE_DOC_START_MARKER));
         objectMapper.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS);
+        objectMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL);
     }
 
     public ToscaTemplate execute() throws FileNotFoundException, JSchException, IOException, ApiException, Exception {
@@ -218,7 +220,7 @@ class CloudStormService {
         }
         CloudCredentialDB cloudStormCredentials = new CloudCredentialDB();
         cloudStormCredentials.setCloudCreds(cloudStormCredentialList);
-        objectMapper.writeValue(new File(credentialsTempInputDirPath + File.separator + "CloudStormCredentials.yml"), cloudStormCredentials);
+        objectMapper.writeValue(new File(credentialsTempInputDirPath + File.separator + "cred.yml"), cloudStormCredentials);
     }
 
     private CredentialInfo getCloudStormCredentialInfo(Credential toscaCredentials, String tmpPath) throws FileNotFoundException {

drip-provisioner/src/main/java/nl/uva/sne/drip/provisioner/Consumer.java

@@ -15,6 +15,7 @@
  */
 package nl.uva.sne.drip.provisioner;
 
+import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.SerializationFeature;
@@ -55,6 +56,7 @@ public class Consumer extends DefaultConsumer {
         logger = Logger.getLogger(Consumer.class.getName());
         this.objectMapper = new ObjectMapper();
         objectMapper.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS);
+        objectMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL);
     }
 
     @Override