set timeout

drip-commons/src/main/java/nl/uva/sne/drip/commons/utils/Converter.java

@@ -15,11 +15,21 @@
  */
 package nl.uva.sne.drip.commons.utils;
 
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.Base64;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import org.json.JSONException;
 import org.json.JSONObject;
+import org.springframework.web.multipart.MultipartFile;
 import org.yaml.snakeyaml.Yaml;
 
 /**
@@ -51,4 +61,37 @@ public class Converter {
         return (Map<String, Object>) object;
     }
 
+    public static String encodeFileToBase64Binary(String fileName) throws IOException {
+        return encode2Bas64(Files.readAllBytes(Paths.get(fileName)));
+    }
+
+    public static void decodeBase64BToFile(String base64, String fileName) throws IOException {
+        byte[] decodedBytrs = Base64.getDecoder().decode(base64);
+        Files.write(Paths.get(fileName), decodedBytrs);
+    }
+
+    public static String getFileMD5(String filePath) throws NoSuchAlgorithmException, IOException {
+        MessageDigest md = MessageDigest.getInstance("MD5");
+        String keyStoreContents = new String(Files.readAllBytes(Paths.get(filePath)));
+        md.update(keyStoreContents.getBytes());
+        byte[] digest = md.digest();
+        return new String(digest, StandardCharsets.UTF_8);
+    }
+
+    public static String encodeFileToBase64Binary(MultipartFile file) throws IOException {
+
+        String originalFileName = file.getOriginalFilename();
+        String name = System.currentTimeMillis() + "_" + originalFileName;
+        byte[] bytes = file.getBytes();
+
+        return encode2Bas64(bytes);
+
+    }
+
+    private static String encode2Bas64(byte[] bytes) {
+
+        byte[] encodedBytes = Base64.getEncoder().encode(bytes);
+        return new String(encodedBytes, StandardCharsets.UTF_8);
+    }
+
 }

drip-commons/src/main/java/nl/uva/sne/drip/commons/utils/ToscaHelper.java

@@ -68,7 +68,10 @@ public class ToscaHelper {
 
     private void init(String sureToscaBasePath) {
         Configuration.getDefaultApiClient().setBasePath(sureToscaBasePath);
+        Configuration.getDefaultApiClient().setConnectTimeout(1200000);
         api = new DefaultApi(Configuration.getDefaultApiClient());
+        System.err.println("ConnectTimeout: " + api.getApiClient().getConnectTimeout());
+
         this.objectMapper = new ObjectMapper(new YAMLFactory().disable(YAMLGenerator.Feature.WRITE_DOC_START_MARKER));
         objectMapper.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS);
         objectMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL);
@@ -186,14 +189,7 @@ public class ToscaHelper {
             if (att == null) {
                 att = new HashMap<>();
             }
-            Map<String, Object> toscaCredential = new HashMap<>();
-            toscaCredential.put("protocol", credential.getProtocol());
-            toscaCredential.put("token_type", credential.getTokenType());
-            toscaCredential.put("token", credential.getToken());
-            toscaCredential.put("keys", credential.getKeys());
-            toscaCredential.put("user", credential.getUser());
-            toscaCredential.put("cloud_provider_name", credential.getCloudProviderName());
-            att.put("credential", toscaCredential);
+            att.put("credential", credential);
             vmTopology.setAttributes(att);
             vmTopologyMap.setNodeTemplate(vmTopology);
             return vmTopologyMap;

drip-manager/src/main/java/nl/uva/sne/drip/api/CredentialApi.java

@@ -14,6 +14,8 @@ import org.springframework.web.bind.annotation.RequestMethod;
 import javax.validation.Valid;
 import java.util.List;
 import nl.uva.sne.drip.model.tosca.Credential;
+import org.springframework.web.bind.annotation.RequestPart;
+import org.springframework.web.multipart.MultipartFile;
 
 @javax.annotation.Generated(value = "io.swagger.codegen.languages.SpringCodegen", date = "2019-12-06T13:31:49.386Z")
 

drip-manager/src/main/java/nl/uva/sne/drip/api/CredentialApiController.java

@@ -3,6 +3,7 @@ package nl.uva.sne.drip.api;
 import nl.uva.sne.drip.model.tosca.Credential;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import io.swagger.annotations.*;
+import java.io.IOException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.http.HttpStatus;
@@ -12,8 +13,11 @@ import org.springframework.web.bind.annotation.RequestBody;
 import javax.validation.Valid;
 import javax.servlet.http.HttpServletRequest;
 import java.util.List;
+import nl.uva.sne.drip.commons.utils.Converter;
 import nl.uva.sne.drip.service.CredentialService;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.web.bind.annotation.RequestPart;
+import org.springframework.web.multipart.MultipartFile;
 
 @javax.annotation.Generated(value = "io.swagger.codegen.languages.SpringCodegen", date = "2019-10-10T17:15:46.465Z")
 
@@ -56,5 +60,4 @@ public class CredentialApiController implements CredentialApi {
         }
         return new ResponseEntity<>(HttpStatus.NOT_IMPLEMENTED);
     }
-
 }

drip-manager/src/main/java/nl/uva/sne/drip/api/DeployerApi.java

@@ -7,19 +7,10 @@ package nl.uva.sne.drip.api;
 
 import io.swagger.annotations.*;
 import org.springframework.http.ResponseEntity;
-import org.springframework.validation.annotation.Validated;
 import org.springframework.web.bind.annotation.PathVariable;
-import org.springframework.web.bind.annotation.RequestBody;
-import org.springframework.web.bind.annotation.RequestHeader;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RequestMethod;
-import org.springframework.web.bind.annotation.RequestParam;
-import org.springframework.web.bind.annotation.RequestPart;
-import org.springframework.web.multipart.MultipartFile;
 
-import javax.validation.Valid;
-import javax.validation.constraints.*;
-import java.util.List;
 @javax.annotation.Generated(value = "io.swagger.codegen.languages.SpringCodegen", date = "2019-12-06T13:31:49.386Z")
 
 @Api(value = "deployer", description = "the deployer API")
@@ -27,18 +18,22 @@ public interface DeployerApi {
 
     @ApiOperation(value = "deploy the software tosca template", nickname = "deployProvisionToscaTemplateByID", notes = "Returns the deployment ID", response = String.class, authorizations = {
         @Authorization(value = "auth", scopes = {
-            @AuthorizationScope(scope = "read:ToscaTemplate", description = "read your topolog template"),
+            @AuthorizationScope(scope = "read:ToscaTemplate", description = "read your topolog template")
+            ,
             @AuthorizationScope(scope = "write:ToscaTemplate", description = "modify topolog template in your account")
         })
-    }, tags={  })
+    }, tags = {})
     @ApiResponses(value = {
-        @ApiResponse(code = 200, message = "successful operation", response = String.class),
-        @ApiResponse(code = 400, message = "Invalid ID supplied"),
-        @ApiResponse(code = 404, message = "ToscaTemplate not found"),
-        @ApiResponse(code = 405, message = "Invalid input") })
+        @ApiResponse(code = 200, message = "successful operation", response = String.class)
+        ,
+        @ApiResponse(code = 400, message = "Invalid ID supplied")
+        ,
+        @ApiResponse(code = 404, message = "ToscaTemplate not found")
+        ,
+        @ApiResponse(code = 405, message = "Invalid input")})
     @RequestMapping(value = "/deployer/deploy/{id}",
-        produces = { "text/plain" }, 
+            produces = {"text/plain"},
             method = RequestMethod.GET)
-    ResponseEntity<String> deployProvisionToscaTemplateByID(@ApiParam(value = "ID of topolog template to deploy",required=true) @PathVariable("id") String id);
+    ResponseEntity<String> deployProvisionToscaTemplateByID(@ApiParam(value = "ID of topolog template to deploy", required = true) @PathVariable("id") String id);
 
 }

drip-manager/src/test/java/nl/uva/sne/drip/service/ServiceTests.java

@@ -28,20 +28,17 @@ import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;
 import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Paths;
-import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
-import java.util.Base64;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.NoSuchElementException;
 import java.util.logging.Level;
 import java.util.logging.Logger;
-import javax.xml.bind.DatatypeConverter;
 import nl.uva.sne.drip.Swagger2SpringBoot;
+import nl.uva.sne.drip.commons.utils.Converter;
 import nl.uva.sne.drip.configuration.MongoConfig;
+import nl.uva.sne.drip.model.cloud.storm.CloudDB;
 import nl.uva.sne.drip.model.tosca.Credential;
 import org.junit.Assert;
 import static org.junit.Assert.assertEquals;
@@ -274,19 +271,10 @@ public class ServiceTests {
     @Test
     public void testCredentialService() throws IOException, NoSuchAlgorithmException {
         Logger.getLogger(ServiceTests.class.getName()).log(Level.INFO, "testCredentialService");
-        MessageDigest md = MessageDigest.getInstance("MD5");
-        md.update(Files.readAllBytes(Paths.get(testCredentialPath)));
-        byte[] digest = md.digest();
-        String fileChecksum = DatatypeConverter
-                .printHexBinary(digest).toUpperCase();
-
-        String keyStore = new String(Files.readAllBytes(Paths.get(testCredentialPath)));
-
-        byte[] encodedBytes = Base64.getEncoder().encode(keyStore.getBytes());
-        String keyStoreEncoded = new String(encodedBytes, "UTF-8");
+        String keyStoreEncoded = Converter.encodeFileToBase64Binary(testCredentialPath);
 
         Credential credential = new Credential();
-        credential.setCloudProviderName("exogeni");
+        credential.setCloudProviderName("ExoGENI");
         Map<String, String> keys = new HashMap<>();
         keys.put("keystore", keyStoreEncoded);
         credential.setKeys(keys);
@@ -294,28 +282,16 @@ public class ServiceTests {
         credential.setTokenType("password");
         credential.setUser("user");
 
-        byte[] decodedBytes = Base64.getDecoder().decode(keys.get("keystore"));
-        md = MessageDigest.getInstance("MD5");
-        md.update(decodedBytes);        
-        digest = md.digest();
-        String credentialChecksum = DatatypeConverter
-                .printHexBinary(digest).toUpperCase();   
-        
-        assertEquals(fileChecksum, credentialChecksum);
-        
-
-        HashMap<Object, Object> att = new HashMap<>();
-        Map<String, Object> toscaCredential = new HashMap<>();
-        toscaCredential.put("protocol", credential.getProtocol());
-        toscaCredential.put("token_type", credential.getTokenType());
-        toscaCredential.put("token", credential.getToken());
-        toscaCredential.put("keys", credential.getKeys());
-        toscaCredential.put("user", credential.getUser());
-        toscaCredential.put("cloud_provider_name", credential.getCloudProviderName());
-        att.put("credential", toscaCredential);
+        String keyStoreEncodedFromCredential = credential.getKeys().get("keystore");
+        assertEquals(keyStoreEncoded, keyStoreEncodedFromCredential);
 
+        String copyTestCredentialPath = ".." + File.separator + "fake_credentials" + File.separator + "copy_of_test-geni.jks";
+        Converter.decodeBase64BToFile(keyStoreEncodedFromCredential, copyTestCredentialPath);
 
+        String keystorFileChecksum = Converter.getFileMD5(testCredentialPath);
+        String keystorFileCopyChecksum = Converter.getFileMD5(copyTestCredentialPath);
 
+        assertEquals(keystorFileChecksum, keystorFileCopyChecksum);
     }
 
     public String saveCredential() {
@@ -354,6 +330,7 @@ public class ServiceTests {
         credentialService.deleteByID(id);
         try {
             Credential res = credentialService.findByID(id);
+            assertNotNull(res);
         } catch (Exception ex) {
             if (!(ex instanceof NoSuchElementException)) {
                 fail(ex.getMessage());

drip-planner/.idea/workspace.xml

 <?xml version="1.0" encoding="UTF-8"?>
 <project version="4">
   <component name="ChangeListManager">
-    <list default="true" id="e478ccae-5352-4e8e-9efb-3f5cda44e877" name="Default Changelist" comment="" />
+    <list default="true" id="e478ccae-5352-4e8e-9efb-3f5cda44e877" name="Default Changelist" comment="">
+      <change beforePath="$PROJECT_DIR$/../drip-commons/src/main/java/nl/uva/sne/drip/commons/utils/Converter.java" beforeDir="false" afterPath="$PROJECT_DIR$/../drip-commons/src/main/java/nl/uva/sne/drip/commons/utils/Converter.java" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/../drip-commons/src/main/java/nl/uva/sne/drip/commons/utils/ToscaHelper.java" beforeDir="false" afterPath="$PROJECT_DIR$/../drip-commons/src/main/java/nl/uva/sne/drip/commons/utils/ToscaHelper.java" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/../drip-manager/src/main/java/nl/uva/sne/drip/api/CredentialApi.java" beforeDir="false" afterPath="$PROJECT_DIR$/../drip-manager/src/main/java/nl/uva/sne/drip/api/CredentialApi.java" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/../drip-manager/src/main/java/nl/uva/sne/drip/api/CredentialApiController.java" beforeDir="false" afterPath="$PROJECT_DIR$/../drip-manager/src/main/java/nl/uva/sne/drip/api/CredentialApiController.java" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/../drip-manager/src/main/java/nl/uva/sne/drip/api/DeployerApi.java" beforeDir="false" afterPath="$PROJECT_DIR$/../drip-manager/src/main/java/nl/uva/sne/drip/api/DeployerApi.java" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/../drip-manager/src/test/java/nl/uva/sne/drip/service/ServiceTests.java" beforeDir="false" afterPath="$PROJECT_DIR$/../drip-manager/src/test/java/nl/uva/sne/drip/service/ServiceTests.java" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/../openAPI/API/CONF-3.0.0-swagger.yaml" beforeDir="false" afterPath="$PROJECT_DIR$/../openAPI/API/CONF-3.0.0-swagger.yaml" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/../sure_tosca-flask-server/sure_tosca/service/tosca_template_service.py" beforeDir="false" afterPath="$PROJECT_DIR$/../sure_tosca-flask-server/sure_tosca/service/tosca_template_service.py" afterDir="false" />
+    </list>
     <option name="SHOW_DIALOG" value="false" />
     <option name="HIGHLIGHT_CONFLICTS" value="true" />
     <option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />

openAPI/API/CONF-3.0.0-swagger.yaml

@@ -234,6 +234,29 @@ paths:
         - read:ToscaTemplate
   /credential:
     post:
+      summary: get credential key file as base64
+      description: get credential key file as base64
+      operationId: getFileasBase64
+      consumes:
+      - multipart/form-data
+      parameters:
+      - name: file
+        in: formData
+        description: credential key file
+        required: true
+        type: file
+      responses:
+        "200":
+          description: successful operation
+          schema:
+            type: string
+        "405":
+          description: Invalid input
+      security:
+      - auth:
+        - write:ToscaTemplate
+        - read:ToscaTemplate
+    put:
       summary: Create credentials
       description: Creates credentials
       operationId: createCredentials
@@ -247,7 +270,7 @@ paths:
         description: Created user object
         required: true
         schema:
-          $ref: '#/definitions/Credentials'
+          $ref: '#/definitions/Credential'
       responses:
         "200":
           description: successful operation
@@ -458,23 +481,29 @@ definitions:
         type: integer
         format: int32
         description: User Status
-  Credentials:
+  Credential:
     type: object
     properties:
       protocol:
         type: string
+        description: The optional protocol name. e.g. http,xauth,oauth2,ssh
       token_type:
         type: string
+        description: 'The required token type. default: password. e.g. basic_auth,X-Auth-Token, bearer, identifier'
       token:
         type: string
+        description: The required token used as a credential for authorization or access to a networked resource. e.g. mypassword, myusername:mypassword,  604bbe45ac7143a79e14f3158df67091, keypair_id
       keys:
         type: object
+        description: The optional list of protocol-specific keys or assertions.
         additionalProperties:
           type: string
       user:
         type: string
+        description: The optional user (name or ID) used for non-token based credentials.
       cloud_provider_name:
         type: string
+        description: The cloud provider name e.g. ec2.
   NodeTemplate:
     type: object
     properties:
@@ -661,19 +690,14 @@ definitions:
         type: array
         items:
           $ref: '#/definitions/CloudDB'
-  CloudDB:
-    type: object
-    properties:
-      cloudProvider:
-        type: string
-      dbInfoFile:
-        type: string
   DBInfo:
     type: object
     properties:
       GlobalEntry:
         type: string
       DCMetaInfo:
+        type: array
+        items:
           $ref: '#/definitions/DCMetaInfo'
   DCMetaInfo:
     type: object
@@ -691,6 +715,8 @@ definitions:
       availability:
         type: string
       VMMetaInfo:
+        type: array
+        items:
           $ref: '#/definitions/VMMetaInfo'
       extraInfo:
         type: object
@@ -706,6 +732,8 @@ definitions:
         type: string
       MEM:
         type: string
+      VMType:
+        type: string
       Price:
         type: string
       DefaultSSHAccount:
@@ -717,6 +745,67 @@ definitions:
         additionalProperties:
           type: object
           properties: {}
+  CloudsStormVMs:
+    type: object
+    properties:
+      VMs:
+        type: array
+        items:
+          $ref: '#/definitions/CloudsStormVM'
+  CloudCredentialDB:
+    type: object
+    properties:
+      cloudCreds:
+        type: array
+        items:
+          $ref: '#/definitions/CloudCred'
+  CredentialInfo:
+    type: object
+    properties:
+      userKeyName:
+        type: string
+      keyAlias:
+        type: string
+      keyPassword:
+        type: string
+      proxyFileName:
+        type: string
+      trustedCertDirName:
+        type: string
+      accessKey:
+        type: string
+      secretKey:
+        type: string
+  NodeTemplateMap:
+    type: object
+    properties:
+      name:
+        type: string
+      nodeTemplate:
+        $ref: '#/definitions/NodeTemplate'
+  Provisioner:
+    type: object
+    properties:
+      name:
+        type: string
+      version:
+        type: string
+      description:
+        type: string
+      tosca_interface_type:
+        type: string
+  CloudsStormInfrasCode:
+    type: object
+    properties:
+      Mode:
+        type: string
+        enum:
+        - LOCAL
+        - CTRL
+      InfrasCodes:
+        type: array
+        items:
+          $ref: '#/definitions/InfrasCode'
   CloudsStormSubTopology:
     type: object
     properties:
@@ -728,6 +817,12 @@ definitions:
         type: string
       status:
         type: string
+        enum:
+        - fresh
+        - running
+        - deleted
+        - failed
+        - stopped
   CloudsStormSubnets:
     type: object
     properties:
@@ -748,3 +843,82 @@ definitions:
         type: string
       address:
         type: string
+  CloudDB:
+    type: object
+    properties:
+      cloudProvider:
+        type: string
+        enum:
+        - EC2
+        - ExoGENI
+        - EGI
+      dbInfoFile:
+        type: string
+  CloudsStormVM:
+    type: object
+    properties:
+      name:
+        type: string
+      nodeType:
+        type: string
+      OStype:
+        type: string
+      script:
+        type: string
+      publicAddress:
+        type: string
+  CloudCred:
+    type: object
+    properties:
+      cloudProvider:
+        type: string
+      credInfoFile:
+        type: string
+  InfrasCode:
+    type: object
+    properties:
+      CodeType:
+        type: string
+        enum:
+        - SEQ
+        - LOOP
+      OpCode:
+        $ref: '#/definitions/OpCode'
+      Count:
+        type: integer
+  OpCode:
+    type: object
+    properties:
+      Operation:
+        type: string
+        enum:
+        - provision
+        - delete
+        - execute
+        - put
+        - get
+        - vscale
+        - hscale
+        - recover
+        - start
+      ObjectType:
+        type: string
+        enum:
+        - SubTopology
+        - VM
+        - REQ
+      Objects:
+        type: string
+      Command:
+        type: string
+      Log:
+        type: boolean
+      Options:
+        $ref: '#/definitions/Options'
+  Options:
+    type: object
+    properties:
+      Src:
+        type: string
+      Dst:
+        type: string

sure_tosca-flask-server/sure_tosca/service/tosca_template_service.py

@@ -2,6 +2,7 @@ import json
 import logging
 import os
 import tempfile
+import time
 import uuid
 from builtins import print
 from functools import reduce
@@ -31,6 +32,15 @@ node_template_db = TinyDB(storage=CachingMiddleware(MemoryStorage))
 dsl_definitions_db = TinyDB(storage=CachingMiddleware(MemoryStorage))
 relationship_template_db = TinyDB(storage=CachingMiddleware(MemoryStorage))
 interface_types_db = TinyDB(storage=CachingMiddleware(MemoryStorage))
+logger = logging.getLogger(__name__)
+if not getattr(logger, 'handler_set', None):
+    logger.setLevel(logging.INFO)
+h = logging.StreamHandler()
+formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+h.setFormatter(formatter)
+logger.addHandler(h)
+logger.handler_set = True
+
 
 root_key = 'root_key'
 
@@ -50,7 +60,7 @@ def query_db(queries, db=None):
                 res_copy.pop(root_key)
                 node = {key: res_copy}
             else:
-                logging.error(str(res) + ' has no ' + root_key)
+                logger.error(str(res) + ' has no ' + root_key)
             updated_results.append(node)
         return updated_results
     return None
@@ -90,17 +100,27 @@ def purge_all_tables():
 def save(file):
     # try:
     # tosca_template_file_path = os.path.join(db_dir_path, file.filename)
+    start = time.time()
+
+
+
+    logger.info("Got request for tosca template")
     purge_all_tables()
     dictionary = yaml.safe_load(file.stream)
-    print(yaml.dump(dictionary))
+    logger.info("tosca template: \n" + str(yaml.dump(dictionary)))
+    # print(yaml.dump(dictionary))
     tosca_template = ToscaTemplate(yaml_dict_tpl=copy.deepcopy(dictionary))
     # all_custom_def = tosca_template.nodetemplates[0].custom_def
     tosca_template_model = ToscaTemplateModel.from_dict(dictionary)
     doc_id = tosca_templates_db.insert(dictionary)
     # tosca_templates_db.close()
+    logger.info("Returning doc_id: " + str(doc_id))
+    end = time.time()
+    elapsed = end - start
+    logger.info("Time elapsed: " + str(elapsed))
     return doc_id
     # except Exception as e:
-    #     logging.error(str(e))
+    #     logger.error(str(e))
     #     return str(e), 400