Merge branch 'develop' into 'staging'

Develop

See merge request !3

bin/docker_push.sh

 #!/bin/bash
 
-docker push qcdis/manager:3.0.0
-docker push qcdis/provisioner:3.0.0
-# docker push qcdis/sure-tosca:3.0.0
-# docker push qcdis/planner:3.0.0
-# docker push qcdis/deployer:3.0.0
+docker push qcdis/manager
+docker push qcdis/provisioner
+docker push qcdis/sure-tosca
+docker push qcdis/planner
+docker push qcdis/deployer

bin/dockerize.sh

@@ -10,21 +10,21 @@ cd provisioner && mvn -Dmaven.test.skip=true dockerfile:build
 [ $status -eq 0 ] && echo "build successful" || exit -1
 cd ../
 echo "----------Building sure_tosca-flask-server Docker--------------"
-cd sure_tosca-flask-server && docker build -t sure-tosca:3.0.0 .
+cd sure_tosca-flask-server && docker build -t sure-tosca .
 [ $status -eq 0 ] && echo "build successful" || exit -1
 echo "----------Building planner Docker--------------"
 cd ../
-cd planner && docker build -t planner:3.0.0 .
+cd planner && docker build -t planner .
 [ $status -eq 0 ] && echo "build successful" || exit -1
 echo "----------Building deployer Docker--------------"
 cd ../
-cd deployer && docker build -t deployer:3.0.0 .
+cd deployer && docker build -t deployer .
 [ $status -eq 0 ] && echo "build successful" || exit -1
 
 
 
-docker tag sure-tosca:3.0.0 qcdis/sure-tosca:3.0.0
-docker tag planner:3.0.0 qcdis/planner:3.0.0
-docker tag deployer:3.0.0 qcdis/deployer:3.0.0
-docker tag manager:3.0.0 qcdis/manager:3.0.0
-docker tag provisioner:3.0.0 qcdis/provisioner:3.0.0
+docker tag sure-tosca qcdis/sure-tosca
+docker tag planner qcdis/planner
+docker tag deployer qcdis/deployer
+docker tag manager:3.0.0 qcdis/manager
+docker tag provisioner:3.0.0 qcdis/provisioner

commons/src/main/java/nl/uva/sne/drip/commons/utils/Converter.java

@@ -21,6 +21,7 @@ import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.UnsupportedEncodingException;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.FileSystem;
 import java.nio.file.FileSystems;
@@ -30,16 +31,26 @@ import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.nio.file.SimpleFileVisitor;
 import java.nio.file.attribute.BasicFileAttributes;
+import java.security.InvalidKeyException;
+import java.security.Key;
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
+import java.util.Arrays;
 import java.util.Base64;
 import java.util.Enumeration;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipFile;
 import java.util.zip.ZipOutputStream;
+import javax.crypto.BadPaddingException;
+import javax.crypto.Cipher;
+import javax.crypto.IllegalBlockSizeException;
+import javax.crypto.NoSuchPaddingException;
+import javax.crypto.spec.SecretKeySpec;
+import nl.uva.sne.drip.model.tosca.Credential;
 import org.json.JSONException;
 import org.json.JSONObject;
 import org.springframework.web.multipart.MultipartFile;
@@ -157,4 +168,62 @@ public class Converter {
         }
     }
 
+    public static String decryptString(String contents, String secret) throws UnsupportedEncodingException, NoSuchAlgorithmException, NoSuchPaddingException, InvalidKeyException, IllegalBlockSizeException, BadPaddingException {
+        SecretKeySpec secretKey = getsecretKey(secret);
+        Cipher cipher = Cipher.getInstance("AES/ECB/PKCS5PADDING");
+        cipher.init(Cipher.DECRYPT_MODE, secretKey);
+        return new String(cipher.doFinal(Base64.getDecoder().decode(contents)));
+
+    }
+
+    public static String encryptString(String contents, String secret) throws UnsupportedEncodingException, NoSuchAlgorithmException, NoSuchPaddingException, InvalidKeyException, IllegalBlockSizeException, BadPaddingException {
+        SecretKeySpec secretKey = getsecretKey(secret);
+        Cipher cipher = Cipher.getInstance("AES/ECB/PKCS5Padding");
+        cipher.init(Cipher.ENCRYPT_MODE, secretKey);
+        return Base64.getEncoder().encodeToString(cipher.doFinal(contents.getBytes("UTF-8")));
+    }
+
+    private static SecretKeySpec getsecretKey(String myKey) throws UnsupportedEncodingException, NoSuchAlgorithmException {
+        MessageDigest sha;
+
+        byte[] key = myKey.getBytes("UTF-8");
+        sha = MessageDigest.getInstance("SHA-1");
+        key = sha.digest(key);
+        key = Arrays.copyOf(key, 16);
+        return new SecretKeySpec(key, "AES");
+    }
+
+    public static Credential encryptCredential(Credential credential, String credentialSecret) throws UnsupportedEncodingException, NoSuchAlgorithmException, NoSuchPaddingException, InvalidKeyException, IllegalBlockSizeException, BadPaddingException {
+
+        Map<String, String> credKeys = credential.getKeys();
+        Set<String> keySet = credKeys.keySet();
+        for (String key : keySet) {
+            String credKey = credKeys.get(key);
+            if (credKey != null) {
+                credKeys.put(key, encryptString(credKey, credentialSecret));
+            }
+        }
+        String token = credential.getToken();
+        if (token != null) {
+            credential.setToken(encryptString(token, credentialSecret));
+        }
+        return credential;
+    }
+
+    public static Credential dencryptCredential(Credential credential, String credentialSecret) throws UnsupportedEncodingException, NoSuchAlgorithmException, NoSuchPaddingException, InvalidKeyException, IllegalBlockSizeException, BadPaddingException {
+        Map<String, String> credKeys = credential.getKeys();
+        Set<String> keySet = credKeys.keySet();
+        for (String key : keySet) {
+            String credKey = credKeys.get(key);
+            if (credKey != null) {
+                credKeys.put(key, decryptString(credKey, credentialSecret));
+            }
+        }
+        String token = credential.getToken();
+        if (token != null) {
+            credential.setToken(decryptString(token, credentialSecret));
+        }
+        return credential;
+    }
+
 }

commons/src/main/resources/application.properties

-message.broker.host=127.0.0.1
-message.broker.username=guest
-message.broker.password=guest
-message.broker.queue.provisioner=provisioner
-message.broker.queue.planner=planner
-message.broker.queue.deployer=deployer
-
-sure-tosca.base.path=http://localhost:8081/tosca-sure/1.0.0

commons/src/test/java/nl/uva/sne/drip/commons/utils/ConverterTest.java

+/*
+ * Copyright 2019 S. Koulouzis
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package nl.uva.sne.drip.commons.utils;
+
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import static org.junit.Assert.*;
+
+/**
+ *
+ * @author S. Koulouzis
+ */
+public class ConverterTest {
+
+    public ConverterTest() {
+    }
+
+    @BeforeClass
+    public static void setUpClass() {
+    }
+
+    @AfterClass
+    public static void tearDownClass() {
+    }
+
+    @Before
+    public void setUp() {
+    }
+
+    @After
+    public void tearDown() {
+    }
+
+    /**
+     * Test of decryptString method, of class Converter.
+     */
+    @Test
+    public void testEncryptDecryptString() throws Exception {
+        System.out.println("decryptString");
+        String contents = "this is very important information";
+        String secret = "top_secret";
+        String expResult = contents;
+        String enc = Converter.encryptString(contents, secret);
+        System.out.println("Encrypted String: " + enc);
+        String result = Converter.decryptString(enc, secret);
+        assertEquals(expResult, result);
+    }
+
+}

deployer/Dockerfile

@@ -28,4 +28,7 @@ RUN pip3 install --no-cache-dir -r requirements.txt
 
 COPY . /usr/src/app
 
-CMD python3 __main__.py $RABBITMQ_HOST deployer
+CMD sed -i "s#http://127.0.0.1:8081/tosca-sure/1.0.0#$SURE_TOSCA_BASE_PATH#g" properties.ini && \
+    sed -i "s#http://127.0.0.1:3000/api#$SEMAPHORE_BASE_PATH#g" properties.ini && \
+    sed -i "s#host = 127.0.0.1#host = $RABBITMQ_HOST#g" properties.ini && \
+    python3 __main__.py $RABBITMQ_HOST deployer

deployer/__main__.py

@@ -8,14 +8,16 @@ import os
 import sys
 import tempfile
 import time
+import traceback
 from threading import Thread
 from time import sleep
-from service.deploy_service import DeployService
-from service.tosca_helper import  ToscaHelper
+
 import pika
-import sure_tosca_client
 import yaml
 
+from service.deploy_service import DeployService
+from service.tosca_helper import ToscaHelper
+
 logger = logging.getLogger(__name__)
 
 done = False
@@ -85,16 +87,28 @@ def handle_delivery(message):
     nodes_pairs = tosca_helper.get_deployment_node_pairs()
 
     deployService = DeployService(semaphore_base_url=semaphore_base_url, semaphore_username=semaphore_username,
-                                  semaphore_password=semaphore_password)
-    for node_pair in nodes_pairs:
-        deployService.deploy(node_pair)
-
-    response = {'toscaTemplate': tosca_template_dict}
-    output_current_milli_time = int(round(time.time() * 1000))
-    response["creationDate"] = output_current_milli_time
-    logger.info("Returning Deployment")
-    logger.info("Output message:" + json.dumps(response))
-    return json.dumps(response)
+                                  semaphore_password=semaphore_password, vms=tosca_helper.get_vms())
+    try:
+        for node_pair in nodes_pairs:
+            updated_node = deployService.deploy(node_pair)
+            if isinstance(updated_node, list):
+                for node in updated_node:
+                    tosca_template_dict = tosca_helper.set_node(node,tosca_template_dict)
+                    logger.info("tosca_template_dict :" + json.dumps(tosca_template_dict))
+            else:
+                tosca_template_dict = tosca_helper.set_node(updated_node, tosca_template_dict)
+                logger.info("tosca_template_dict :" + json.dumps(tosca_template_dict))
+
+        response = {'toscaTemplate': tosca_template_dict}
+        output_current_milli_time = int(round(time.time() * 1000))
+        response["creationDate"] = output_current_milli_time
+        logger.info("Returning Deployment")
+        logger.info("Output message:" + json.dumps(response))
+        return json.dumps(response)
+    except Exception as e:
+        track = traceback.format_exc()
+        print(track)
+        raise
 
 
 def threaded_function(args):

deployer/build/lib/test/test_deployer.py

@@ -39,7 +39,6 @@ class TestDeployer(unittest.TestCase):
 
 
             sure_tosca_base_url = 'http://127.0.0.1:8081/tosca-sure/1.0.0'
-            polemarch_base_url='http://127.0.0.1:30001/api/v2'
             semaphore_base_url = 'http://127.0.0.1:3000/api'
             tosca_service_is_up = ToscaHelper.service_is_up(sure_tosca_base_url)
             semaphore_is_up = ToscaHelper.service_is_up(semaphore_base_url)

deployer/service/ansible_service.py

@@ -6,7 +6,7 @@ import datetime
 import yaml
 from semaphore_client.semaphore_helper import SemaphoreHelper
 
-yaml.Dumper.ignore_aliases = lambda *args : True
+yaml.Dumper.ignore_aliases = lambda *args: True
 
 logger = logging.getLogger(__name__)
 if not getattr(logger, 'handler_set', None):
@@ -20,21 +20,20 @@ if not getattr(logger, 'handler_set', None):
 
 class AnsibleService:
 
-    def __init__(self, semaphore_base_url=None,semaphore_username=None,semaphore_password=None):
+    def __init__(self, semaphore_base_url=None, semaphore_username=None, semaphore_password=None):
         self.semaphore_base_url = semaphore_base_url
         self.semaphore_username = semaphore_username
         self.semaphore_password = semaphore_password
-        self.semaphore_helper = SemaphoreHelper(self.semaphore_base_url, self.semaphore_username, self.semaphore_password)
+        self.semaphore_helper = SemaphoreHelper(self.semaphore_base_url, self.semaphore_username,
+                                                self.semaphore_password)
         self.repository_id = None
         self.template_id = None
 
-
-
-    def execute(self,nodes_pair):
-        vms = nodes_pair[0]
+    def execute(self, nodes_pair, interface_type, vms, env_vars=None):
         application = nodes_pair[1]
         name = application.name
         desired_state = None
+        tasks_outputs = {}
         interfaces = application.node_template.interfaces
         if 'current_state' in application.node_template.attributes:
             current_state = application.node_template.attributes['current_state']
@@ -43,31 +42,45 @@ class AnsibleService:
 
         if desired_state:
             now = datetime.datetime.now()
-            project_id = self.semaphore_helper.create_project(application.name+'_'+str(now))
-            inventory_contents = yaml.dump( self.build_yml_inventory(vms),default_flow_style=False)
+            project_id = self.semaphore_helper.create_project(application.name + '_' + str(now))
+            inventory_contents = yaml.dump(self.build_yml_inventory(vms), default_flow_style=False)
             private_key = self.get_private_key(vms)
             key_id = self.semaphore_helper.create_ssh_key(application.name, project_id, private_key)
-            inventory_id = self.semaphore_helper.create_inventory(application.name, project_id, key_id,inventory_contents)
+            inventory_id = self.semaphore_helper.create_inventory(application.name, project_id, key_id,
+                                                                  inventory_contents)
             if 'RUNNING' == desired_state:
-                standard = interfaces['Standard']
-                create = standard['create']
+                interface = interfaces[interface_type]
+                create = interface['create']
                 inputs = create['inputs']
                 git_url = inputs['repository']
                 playbook_names = inputs['resources']
                 for playbook_name in playbook_names:
-                    task_id = self.run_task(name, project_id, key_id, git_url, inventory_id, playbook_name)
+                    environment_id = None
+                    if env_vars:
+                        environment_id = self.semaphore_helper.create_environment(project_id, name, env_vars)
+                    task_id = self.run_task(name, project_id, key_id, git_url, inventory_id, playbook_name,
+                                            environment_id=environment_id)
                     if self.semaphore_helper.get_task(project_id, task_id).status != 'success':
                         break
+                    # logger.info('playbook: ' + playbook_name + ' task_id: ' + str(task_id))
+                    tasks_outputs[task_id] = self.semaphore_helper.get_task_outputs(project_id, task_id)
 
-                if self.semaphore_helper.get_task(project_id,task_id).status == 'success':
-                    configure = standard['configure']
+                if 'configure' in interface and self.semaphore_helper.get_task(project_id, task_id).status == 'success':
+                    configure = interface['configure']
                     inputs = configure['inputs']
                     git_url = inputs['repository']
                     playbook_names = inputs['resources']
                     for playbook_name in playbook_names:
-                        task_id = self.run_task(name, project_id, key_id, git_url, inventory_id, playbook_name)
+                        environment_id = None
+                        if env_vars:
+                            environment_id = self.semaphore_helper.create_environment(project_id, name, env_vars)
+                        task_id = self.run_task(name, project_id, key_id, git_url, inventory_id, playbook_name,
+                                                environment_id=environment_id)
                         if self.semaphore_helper.get_task(project_id, task_id).status != 'success':
                             break
+                        # logger.info('playbook: ' + playbook_name + ' task_id: ' + str(task_id))
+                        tasks_outputs[task_id] = self.semaphore_helper.get_task_outputs(project_id, task_id)
+            return tasks_outputs
 
     def build_yml_inventory(self, vms):
         # loader = DataLoader()
@@ -75,7 +88,7 @@ class AnsibleService:
         # variable_manager = VariableManager()
         inventory = {}
         all = {}
-        vars = {'ansible_ssh_common_args':'-o StrictHostKeyChecking=no'}
+        vars = {'ansible_ssh_common_args': '-o StrictHostKeyChecking=no'}
         vars['ansible_ssh_user'] = vms[0].node_template.properties['user_name']
         children = {}
         for vm in vms:
@@ -87,7 +100,7 @@ class AnsibleService:
             else:
                 hosts = children[role]
             host = {}
-            host[public_ip] =  vars
+            host[public_ip] = vars
             hosts['hosts'] = host
             children[role] = hosts
             # inventory.add_group(role)
@@ -100,24 +113,21 @@ class AnsibleService:
         private_key = vms[0].node_template.attributes['user_key_pair']['keys']['private_key']
         return base64.b64decode(private_key).decode('utf-8').replace(r'\n', '\n')
 
-    def run_task(self, name, project_id, key_id, git_url, inventory_id, playbook_name):
-        logger.info('project_id: '+str(project_id)+ ' task name: ' + str(name)+ ' git url: '+git_url+' playbook: '+playbook_name)
+    def run_task(self, name, project_id, key_id, git_url, inventory_id, playbook_name, environment_id=None):
+        logger.info('project_id: ' + str(project_id) + ' task name: ' + str(
+            name) + ' git url: ' + git_url + ' playbook: ' + playbook_name)
         self.repository_id = self.semaphore_helper.create_repository(name, project_id, key_id, git_url)
         template_id = self.semaphore_helper.create_template(project_id, key_id, inventory_id, self.repository_id,
                                                             playbook_name)
-        task_id = self.semaphore_helper.execute_task(project_id, template_id, playbook_name)
+        task_id = self.semaphore_helper.execute_task(project_id, template_id, playbook_name,
+                                                     environment_id=environment_id)
         task = self.semaphore_helper.get_task(project_id, task_id)
-        last_output = ''
+        last_status = ''
         while task.status == 'waiting' or task.status == 'running':
             task = self.semaphore_helper.get_task(project_id, task_id)
-            logger.info('task name: '+name+ ' task status: ' + str(task.status))
-            task_outputs = self.semaphore_helper.get_task_outputs(project_id, task_id)
-            this_output = task_outputs[len(task_outputs)-1].output.replace(r'\n', '\n').replace(r'\r', '\r')
-            if last_output != this_output:
-                logger.info('task output: ' + str(this_output))
-            last_output = this_output
-
-            # logger.info('task output: ' + str(latask name:st_output))
+            this_status = task.status
+            if last_status != this_status:
+                logger.info('task name: ' + name + ', task status: ' + str(task.status))
+            last_status = this_status
             sleep(3)
         return task_id
-

deployer/service/deploy_service.py

-from service import tosca_helper, ansible_service
+import json
+import logging
+
+from service import tosca_helper
 from service.ansible_service import AnsibleService
 
+logger = logging.getLogger(__name__)
+if not getattr(logger, 'handler_set', None):
+    logger.setLevel(logging.INFO)
+    h = logging.StreamHandler()
+    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+    h.setFormatter(formatter)
+    logger.addHandler(h)
+    logger.handler_set = True
 
 class DeployService:
 
     def __init__(self, polemarch_base_url=None,polemarch_username=None,polemarch_password=None,
-                 semaphore_base_url=None,semaphore_username=None,semaphore_password=None):
+                 semaphore_base_url=None,semaphore_username=None,semaphore_password=None, vms=None):
         # self.polemarch_base_url = polemarch_base_url
         # self.polemarch_username=polemarch_username
         # self.polemarch_password = polemarch_password
         self.semaphore_base_url = semaphore_base_url
         self.semaphore_username = semaphore_username
         self.semaphore_password = semaphore_password
+        self.vms = vms
+        for vm in vms:
+            if vm.node_template.attributes['role'] == 'master':
+                self.master_ip = vm.node_template.attributes['public_ip']
+                break
 
     def deploy(self,nodes_pair):
         target = nodes_pair[0]
         source = nodes_pair[1]
-
         interface_types = tosca_helper.get_interface_types(source)
         if interface_types:
+            ansible_service = AnsibleService(self.semaphore_base_url, self.semaphore_username, self.semaphore_password)
+            env_vars = self.get_env_vars(nodes_pair)
             if 'Standard' in interface_types:
-                ansible_service = AnsibleService(self.semaphore_base_url,self.semaphore_username,self.semaphore_password)
-                ansible_service.execute(nodes_pair)
-            else:
-                print(interface_types)
+                task_outputs =  ansible_service.execute(nodes_pair, 'Standard', self.vms, env_vars=env_vars)
+                source = self.set_attributes(task_outputs,source)
+            if 'Kubernetes' in interface_types:
+                task_outputs = ansible_service.execute(nodes_pair, 'Kubernetes', self.vms, env_vars=env_vars)
+                source = self.set_attributes(task_outputs,source)
+
+        return source
+
+    def get_env_vars(self, nodes_pair):
+        target = nodes_pair[0]
+        source = nodes_pair[1]
+        env_vars = {'K8s_NAMESPACE': 'default'}
+        if source.node_template.type == 'tosca.nodes.QC.Container.Application.Docker':
+            env_vars['DOCKER_IMAGE'] = source.node_template.artifacts['image']['file']
+            env_vars['DOCKER_SERVICE_NAME'] = source.name
+            env_vars['CONTAINER_PORT'] = source.node_template.properties['ports'][0].split(':')[1]
+        return env_vars
+
+    def set_attributes(self, task_outputs,source):
+        # target = nodes_pair[0]
+        # source = nodes_pair[1]
+        if source.node_template.type == 'tosca.nodes.QC.docker.Orchestrator.Kubernetes':
+            source = self.set_kubernetes_attributes(source=source,task_outputs=task_outputs)
+        if source.node_template.type == 'tosca.nodes.QC.Container.Application.Docker':
+            source = self.set_docker_attributes(source=source, task_outputs=task_outputs)
+        # lst = list(nodes_pair)
+        # lst[1] = source
+        # nodes_pair = tuple(lst)
+        return source
+
+
+
+    def parse_ansible_var(self,var_name, output_array):
+        index = 0
+        start_index = -1
+        end_index = -1
+        for out in output_array:
+            index += 1
+            if 'TASK' in out.output or 'PLAY RECAP' in out.output:
+                if start_index > -1:
+                    end_index = index - 1
+                    break
+            if start_index <=-1 and '"' + var_name + '":' in out.output:
+                start_index = index - 1
+        if start_index <= -1:
+            return None
+        ansible_var = output_array[start_index:end_index]
+        json_ansible_var = '{'
+        for item in ansible_var:
+            json_ansible_var = json_ansible_var + item.output
+        logger.info('found '+var_name +': '+ str(json_ansible_var))
+        return json.loads(json_ansible_var)
 
+    def get_dashboard_token(self, k8s_dashboard_token):
+        k8s_dashboard_token = k8s_dashboard_token['k8s_dashboard_token']
+        # if 'resources' in k8s_secrets:
+        #     return self.get_secret_from_k8s_info(k8s_secrets)
+        if 'stdout' in k8s_dashboard_token:
+            return self.get_secret_from_stdout(k8s_dashboard_token)
+
+
+    def get_service_port(self, k8s_services, service_name,port_type):
+        resources = k8s_services['k8s_services']['resources']
+        for resource in resources:
+            name = resource['metadata']['name']
+            if name == service_name:
+                ports = resource['spec']['ports']
+                for port in ports:
+                    if port_type in port:
+                        return port[port_type]
         return None
+
+    def get_secret_from_k8s_info(self, k8s_secrets):
+        resources = k8s_secrets['resources']
+        for resource in resources:
+            metadata = resource['metadata']
+            if 'admin-user-token' in  metadata['name']:
+                dashboard_token = resource['data']['token']
+                logger.info('found dashboard_token: ' + str(dashboard_token))
+                return resource['data']['token']
+        return None
+
+    def get_secret_from_stdout(self, k8s_dashboard_token):
+        return k8s_dashboard_token['stdout'].replace('token:     ', '')
+
+    def set_kubernetes_attributes(self, source=None,task_outputs=None):
+        attributes = source.node_template.attributes
+        if 'tokens' not in attributes:
+            tokens = []
+            attributes['tokens'] = tokens
+        else:
+            tokens = attributes['tokens']
+
+        if 'dashboard_url' not in source.node_template.attributes:
+            dashboard_url = ''
+            attributes['dashboard_url'] = dashboard_url
+        else:
+            dashboard_url = attributes['dashboard_url']
+
+        k8s_dashboard_token = None
+        k8s_services = None
+        for task_output_key in task_outputs:
+            task_output = task_outputs[task_output_key]
+            if not k8s_dashboard_token:
+                k8s_dashboard_token = self.parse_ansible_var('k8s_dashboard_token', task_output)
+            if not k8s_services:
+                k8s_services = self.parse_ansible_var('k8s_services', task_output)
+            if k8s_services and k8s_dashboard_token:
+                credential = {'token_type': 'k8s_dashboard_token'}
+                credential['token'] = self.get_dashboard_token(k8s_dashboard_token)
+                tokens.append(credential)
+
+                service_port = self.get_service_port(k8s_services, 'kubernetes-dashboard', 'nodePort')
+                dashboard_url = 'https://' + self.master_ip + ':' + str(service_port)
+                attributes['dashboard_url'] = dashboard_url
+                logger.info('source.node_template.attributes: ' + str(attributes))
+                return source
+
+    def set_docker_attributes(self, source, task_outputs):
+        attributes = source.node_template.attributes
+        if 'service_url' not in source.node_template.attributes:
+            service_url = ''
+            attributes['service_url'] = service_url
+        for task_output_key in task_outputs:
+            task_output = task_outputs[task_output_key]
+            k8s_services = self.parse_ansible_var('k8s_services', task_output)
+            service_port = self.get_service_port(k8s_services, source.name, 'nodePort')
+            if service_port:
+                service_url = 'http://' + self.master_ip + ':' + str(service_port)
+                attributes['service_url'] = service_url
+                logger.info('source.node_template.attributes: ' + str(attributes))
+            return source

deployer/service/tosca_helper.py

+import copy
 import os
 import sys
 import urllib.request
 
-from sure_tosca_client import Configuration, ApiClient
+from sure_tosca_client import Configuration, ApiClient, NodeTemplate
 from sure_tosca_client.api import default_api
 
 
@@ -28,7 +29,6 @@ class ToscaHelper:
         return api
 
     def get_interface_types(target):
-        print(target.node_template.interfaces)
         interface_types = []
         for interface in target.node_template.interfaces:
             interface_types.append(interface)
@@ -49,7 +49,7 @@ class ToscaHelper:
                 # So we explicitly get the VMs
                 # I don't like this solution but I can't think of something better.
                 if related_node.node_template.type == 'tosca.nodes.QC.VM.topology':
-                    vms = self.tosca_client.get_node_templates(self.doc_id,type_name='tosca.nodes.QC.VM.Compute')
+                    vms = self.get_vms()
                     related_node = vms
                 pair = (related_node, node)
                 nodes_pairs.append(pair)
@@ -70,6 +70,16 @@ class ToscaHelper:
 
         return True
 
+    def get_vms(self):
+        return self.tosca_client.get_node_templates(self.doc_id, type_name='tosca.nodes.QC.VM.Compute')
+
+    def set_node(self, updated_node, tosca_template_dict):
+        node_templates = tosca_template_dict['topology_template']['node_templates']
+        for node_name in node_templates:
+            if node_name == updated_node.name:
+                node_templates[node_name] = updated_node.node_template.to_dict()
+                return tosca_template_dict
+
 
 def get_interface_types(node):
     interface_type_names = []

deployer/test/test_deployer.py

@@ -39,7 +39,6 @@ class TestDeployer(unittest.TestCase):
 
 
             sure_tosca_base_url = 'http://127.0.0.1:8081/tosca-sure/1.0.0'
-            polemarch_base_url='http://127.0.0.1:30001/api/v2'
             semaphore_base_url = 'http://127.0.0.1:3000/api'
             tosca_service_is_up = ToscaHelper.service_is_up(sure_tosca_base_url)
             semaphore_is_up = ToscaHelper.service_is_up(semaphore_base_url)
@@ -53,8 +52,9 @@ class TestDeployer(unittest.TestCase):
                 self.assertIsNotNone(nodes_pairs)
 
                 username = 'admin'
-                deployService = DeployService(polemarch_base_url=polemarch_base_url,polemarch_username=username,polemarch_password='admin',
-                                              semaphore_base_url=semaphore_base_url,semaphore_username=username,semaphore_password='password')
+                deployService = DeployService(polemarch_username=username,polemarch_password='admin',
+                                              semaphore_base_url=semaphore_base_url,semaphore_username=username,
+                                              semaphore_password='password')
                 for node_pair in nodes_pairs:
                     deployService.deploy(node_pair)
 

docker-compose/docker-compose.yml

 version: '3'
 services:
 
-  #nginx:
-    #image: nginx
-    #volumes:
-      #- ./nginx.conf:/etc/nginx/nginx.conf
-      #- ./cert.pem:/etc/nginx/cert.pem
-      #- ./privkey.pem:/etc/nginx/privkey.pem
-      ##- ./www:/data/www
-    #ports:
-    #- "80:80"
-    #- "443:443"
+  nginx:
+    image: nginx
+    volumes:
+      - ./nginx.conf:/etc/nginx/nginx.conf
+      - ./cert.pem:/etc/nginx/cert.pem
+      - ./privkey.pem:/etc/nginx/privkey.pem
+      #- ./www:/data/www
+    ports:
+    - "80:80"
+    - "443:443"
     
   rabbit:
     image: rabbitmq:3.8-management
@@ -55,21 +55,22 @@ services:
       - "27017:27017"               
     
       
-  #manager:
-    #depends_on:
-      #- rabbit
-      #- mongo
-      #- sure-tosca
-    #image: qcdis/manager:3.0.0 
-    #environment:
-      #RABBITMQ_HOST: rabbit
-      #MONGO_HOST: mongo
-      #SURE_TOSCA_BASE_PATH: http://sure-tosca:8081/tosca-sure/1.0.0
-    #ports:
-      #- "8080:8080"       
+  manager:
+    depends_on:
+      - rabbit
+      - mongo
+      - sure-tosca
+    image: qcdis/manager 
+    environment:
+      RABBITMQ_HOST: rabbit
+      MONGO_HOST: mongo
+      SURE_TOSCA_BASE_PATH: http://sure-tosca:8081/tosca-sure/1.0.0
+      CREDENTIAL_SECRET: 123
+    ports:
+      - "8080:8080"       
             
   sure-tosca:
-    image: qcdis/sure-tosca:3.0.0
+    image: qcdis/sure-tosca
     ports: 
       - "8081:8081"
     
@@ -77,27 +78,32 @@ services:
     depends_on:
       - rabbit
       - sure-tosca
-    image: qcdis/planner:3.0.0 
+    image: qcdis/planner 
     environment:
       RABBITMQ_HOST: rabbit
     
-  #provisioner:
-    #depends_on:
-      #- rabbit
-      #- sure-tosca
-    #image: qcdis/provisioner:3.0.0 
-    #environment:
-      #RABBITMQ_HOST: rabbit      
-      #SURE_TOSCA_BASE_PATH: http://sure-tosca:8081/tosca-sure/1.0.0
+  provisioner:
+    depends_on:
+      - rabbit
+      - sure-tosca
+    image: qcdis/provisioner 
+    environment:
+      RABBITMQ_HOST: rabbit      
+      SURE_TOSCA_BASE_PATH: http://sure-tosca:8081/tosca-sure/1.0.0
+      CLOUD_STORM_SECRET: 456
+      CREDENTIAL_SECRET: 123      
     
-  #deployer:
-    #depends_on:
-      #- rabbit
-      #- sure-tosca
-    #image: qcdis/deployer:3.0.0 
-    #environment:
-      #RABBITMQ_HOST: rabbit           
-      #SURE_TOSCA_BASE_PATH: http://sure-tosca:8081/tosca-sure/1.0.0
+  deployer:
+    depends_on:
+      - rabbit
+      - sure-tosca
+    image: qcdis/deployer 
+    environment:
+      RABBITMQ_HOST: rabbit           
+      SURE_TOSCA_BASE_PATH: http://sure-tosca:8081/tosca-sure/1.0.0
+      SEMAPHORE_BASE_PATH: http://semaphore:3000/api
+      
+          
       
 
     

docker-compose/nginx.conf

 worker_processes auto;
+  
 
 events {
   worker_connections 1024;
@@ -7,19 +8,17 @@ events {
 }
 
 http {
+
+  proxy_connect_timeout       9200;
+  proxy_send_timeout          9200;
+  proxy_read_timeout          9200;
+  send_timeout                9200;
+  
   tcp_nodelay on;
 
   # this is necessary for us to be able to disable request buffering in all cases
   proxy_http_version 1.1;
 
- # upstream semaphore {
- #   server semaphore:3000;
- # }
- 
-#  upstream manager {
-#    server manager:8080;
-#  } 
-
   server {
   	listen [::]:80 default_server;
   	server_name _;
@@ -60,17 +59,10 @@ http {
       proxy_buffering off;
       proxy_request_buffering off;
     }
-
-#    location /api/ws {
-#      proxy_pass http://semaphore:3000/api/ws;
-#      proxy_http_version 1.1;
-#      proxy_set_header Upgrade $http_upgrade;
-#      proxy_set_header Connection "upgrade";
-#      proxy_set_header Origin "";
-#    }
     
-    location /manger/ {
-      proxy_pass http://manager:8080/manger/;
+    
+    location /manager {
+      proxy_pass http://manager:8080/manager;
       proxy_set_header Host $http_host;
       proxy_set_header X-Real-IP $remote_addr;
       proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
@@ -81,8 +73,8 @@ http {
       proxy_request_buffering off;
     }    
     
-    location /manger/swagger-ui.html {
-      proxy_pass http://manager:8080/swagger-ui.html/;
+    location /swagger-ui.html {
+      proxy_pass http://manager:8080/swagger-ui.html;
       proxy_set_header Host $http_host;
       proxy_set_header X-Real-IP $remote_addr;
       proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
@@ -93,8 +85,8 @@ http {
       proxy_request_buffering off;
     }
     
-    location /tosca-sure/1.0.0/ {
-      proxy_pass http://sure-tosca:8081/tosca-sure/1.0.0/;
+    location /tosca-sure/1.0.0 {
+      proxy_pass http://sure-tosca:8081/tosca-sure/1.0.0;
       proxy_set_header Host $http_host;
       proxy_set_header X-Real-IP $remote_addr;
       proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;

example_messages/message_provision_request.json

 {
   "owner": "user",
-  "creationDate": 1585660972026,
+  "creationDate": 1591357712632,
   "toscaTemplate": {
     "tosca_definitions_version": "tosca_simple_yaml_1_0",
     "tosca_default_namespace": null,
@@ -11,7 +11,7 @@
       "node_templates": {
         "compute": {
           "properties": {
-            "disk_size": "10000 MB",
+            "disk_size": "20000 MB",
             "mem_size": "1000 MB",
             "num_cores": 1,
             "os": "Ubuntu 18.04",
@@ -26,7 +26,7 @@
         },
         "compute_1": {
           "properties": {
-            "disk_size": "10000 MB",
+            "disk_size": "20000 MB",
             "mem_size": "1000 MB",
             "num_cores": 1,
             "os": "Ubuntu 18.04",
@@ -39,7 +39,11 @@
           },
           "type": "tosca.nodes.QC.VM.Compute"
         },
-        "kubernetes": {
+        "glusterfs": {
+          "properties": {
+            "min_masters_num": 1,
+            "min_workers_num": 1
+          },
           "requirements": [
             {
               "host": {
@@ -53,25 +57,60 @@
             "Standard": {
               "configure": {
                 "inputs": {
-                  "playbook": "k8s_dashboard.yaml",
-                  "repository": "https://github.com/skoulouzis/playbooks.git"
+                  "repository": "https://github.com/QCDIS/mysome_glusterfs.git",
+                  "resources": [
+                    "002.setup_glusterfs_infra.yml",
+                    "003.setup_glusterfs_cluster.yml",
+                    "004.mount_glusterfs.yml"
+                  ]
                 }
               },
               "create": {
                 "inputs": {
-                  "playbook": "install_k8s.yaml",
-                  "repository": "https://github.com/skoulouzis/playbooks.git"
+                  "repository": "https://github.com/QCDIS/playbooks.git",
+                  "resources": [
+                    "ec2/ec2_mount.yaml",
+                    "install_glusterfs_prerequisites.yaml"
+                  ]
                 }
-              },
-              "delete": {
+              }
+            }
+          },
+          "type": "tosca.nodes.QC.Application.GlusterFS"
+        },
+        "tic": {
+          "requirements": [
+            {
+              "storage": {
+                "capability": "tosca.capabilities.QC.GlusterFS",
+                "node": "glusterfs",
+                "relationship": "tosca.relationships.DependsOn"
+              }
+            }
+          ],
+          "interfaces": {
+            "Standard": {
+              "create": {
                 "inputs": {
-                  "playbook": "k8s_delete.yaml",
-                  "repository": "https://github.com/skoulouzis/playbooks.git"
+                  "repository": "https://github.com/bityoga/fabric_as_code.git",
+                  "resources": [
+                    "011.initialize_hosts.yml",
+                    "012.prepare_docker_images.yml",
+                    "013.mount_fs.yml",
+                    "014.spawn_swarm.yml",
+                    "015.deploy_swarm_visualizer.yml",
+                    "016.deploy_portainer.yml",
+                    "100.deploy_ca.yml",
+                    "101.deploy_orderer.yml",
+                    "102.deploy_peers.yml",
+                    "103.deploy_cli.yml",
+                    "104.deploy_hlf_explorer.yml"
+                  ]
                 }
               }
             }
           },
-          "type": "tosca.nodes.QC.docker.Orchestrator.Kubernetes"
+          "type": "tosca.nodes.QC.Application.TIC"
         },
         "topology": {
           "properties": {
@@ -133,9 +172,9 @@
             "credential": {
               "cloud_provider_name": "EC2",
               "keys": {
-                "aws_access_key_id": "XXXXXXXXXXXX"
+                "aws_access_key_id": "BGCAc9htRRjcZx5uZ6aWnRo67tNAgk7M26DwuN3cZvypnCt6V56U9RbEuRpmXbER"
               },
-              "token": "XXXXXXXXXXXXX",
+              "token": "BGCAc9htRRjcZx5uZ6aWnRo67tNAgk7M26DwuN3cZvypnCt6V56U9RbEuRpmXbER",
               "token_type": "access_key"
             },
             "desired_state": "RUNNING"
@@ -146,30 +185,6 @@
               "type": "string"
             }
           }
-        },
-        "ws-pema": {
-          "properties": {
-            "ports": [
-              "30001:8080"
-            ]
-          },
-          "requirements": [
-            {
-              "host": {
-                "capability": "tosca.capabilities.QC.docker.Orchestrator",
-                "node": "kubernetes",
-                "relationship": "tosca.relationships.HostedOn"
-              }
-            }
-          ],
-          "type": "tosca.nodes.QC.Container.Application.Docker",
-          "artifacts": {
-            "image": {
-              "file": "alogo53/ws-pema-lifewatch",
-              "repository": "docker_hub",
-              "type": "tosca.artifacts.Deployment.Image.Container.Docker"
-            }
-          }
         }
       },
       "relationship_templates": null,
@@ -183,19 +198,19 @@
     "description": "TOSCA example",
     "imports": [
       {
-        "nodes": "https://raw.githubusercontent.com/skoulouzis/DRIP/develop/TOSCA/types/nodes.yaml"
+        "nodes": "https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/types/nodes.yaml"
       },
       {
-        "data": "https://raw.githubusercontent.com/skoulouzis/CONF/develop/TOSCA/types/data.yml"
+        "data": "https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/types/data.yml"
       },
       {
-        "capabilities": "https://raw.githubusercontent.com/skoulouzis/DRIP/develop/TOSCA/types/capabilities.yaml"
+        "capabilities": "https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/types/capabilities.yaml"
       },
       {
-        "policies": "https://raw.githubusercontent.com/skoulouzis/DRIP/develop/TOSCA/types/policies.yaml"
+        "policies": "https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/types/policies.yaml"
       },
       {
-        "interfaces": "https://raw.githubusercontent.com/skoulouzis/DRIP/develop/TOSCA/types/interfaces.yml"
+        "interfaces": "https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/types/interfaces.yml"
       }
     ],
     "dsl_definitions": null,

k8s/CONF/deployer-deployment.yaml

@@ -33,7 +33,9 @@ spec:
           value: rabbit
         - name: SURE_TOSCA_BASE_PATH
           value: http://sure-tosca:8081/tosca-sure/1.0.0
-        image: qcdis/deployer:3.0.0
+        - name: SEMAPHORE_BASE_PATH
+          value: http://semaphore:3000/api                    
+        image: qcdis/deployer
         name: deployer
         imagePullPolicy: Always
         resources: {}

k8s/CONF/manager-deployment.yaml

@@ -35,6 +35,8 @@ spec:
           value: rabbit
         - name: SURE_TOSCA_BASE_PATH
           value: http://sure-tosca:8081/tosca-sure/1.0.0
+        - name: CREDENTIAL_SECRET
+          value: top_secret
         image: qcdis/manager:3.0.0
         name: manager
         imagePullPolicy: Always

k8s/CONF/manager-deployment.yaml.template

+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  namespace: conf
+  annotations:
+    kompose.cmd: kompose convert
+    kompose.version: 1.16.0 (0c01309)
+  creationTimestamp: null
+  labels:
+    io.kompose.service: manager
+  name: manager
+spec:
+  selector:
+    matchLabels:
+      io.kompose.service: manager
+  replicas: 1
+  strategy:
+        type: RollingUpdate
+        rollingUpdate:
+            maxSurge: 50%
+            maxUnavailable: 50%
+  minReadySeconds: 10
+  revisionHistoryLimit: 3
+  template:
+    metadata:
+      creationTimestamp: null
+      labels:
+        io.kompose.service: manager
+    spec:
+      containers:
+      - env:
+        - name: MONGO_HOST
+          value: mongo
+        - name: RABBITMQ_HOST
+          value: rabbit
+        - name: SURE_TOSCA_BASE_PATH
+          value: http://sure-tosca:8081/tosca-sure/1.0.0
+        - name: CREDENTIAL_SECRET
+          value: top_secret
+        image: qcdis/manager:3.0.0
+        name: manager
+        imagePullPolicy: Always
+        ports:
+        - containerPort: 8080
+        resources: {}
+      restartPolicy: Always
+status: {}

k8s/CONF/nginx-service.yaml

@@ -8,10 +8,6 @@ metadata:
 spec:
   type: NodePort  
   ports:
-  - port: 80
-    nodePort: 30000
-    protocol: TCP
-    name: http
   - port: 443
     nodePort: 30001
     protocol: TCP

k8s/CONF/planner-deployment.yaml

@@ -31,7 +31,7 @@ spec:
       - env:
         - name: RABBITMQ_HOST
           value: rabbit
-        image: qcdis/planner:3.0.0
+        image: qcdis/planner
         name: planner
         imagePullPolicy: Always
         resources: {}

k8s/CONF/provisioner-deployment.yaml.template

+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  namespace: conf
+  annotations:
+    kompose.cmd: kompose convert
+    kompose.version: 1.16.0 (0c01309)
+  creationTimestamp: null
+  labels:
+    io.kompose.service: provisioner
+  name: provisioner
+spec:
+  selector:
+    matchLabels:
+      io.kompose.service: provisioner
+  replicas: 1
+  strategy:
+        type: RollingUpdate
+        rollingUpdate:
+            maxSurge: 50%
+            maxUnavailable: 50%
+  minReadySeconds: 10
+  revisionHistoryLimit: 3
+  template:
+    metadata:
+      creationTimestamp: null
+      labels:
+        io.kompose.service: provisioner
+    spec:
+      containers:
+      - env:
+        - name: RABBITMQ_HOST
+          value: rabbit
+        - name: SURE_TOSCA_BASE_PATH
+          value: http://sure-tosca:8081/tosca-sure/1.0.0
+        image: qcdis/provisioner:3.0.0
+        name: provisioner
+        imagePullPolicy: Always
+        resources: {}
+      restartPolicy: Always
+status: {}

k8s/CONF/sure-tosca-deployment.yaml

@@ -28,7 +28,7 @@ spec:
         io.kompose.service: sure-tosca
     spec:
       containers:
-      - image: qcdis/sure-tosca:3.0.0
+      - image: qcdis/sure-tosca
         name: sure-tosca
         imagePullPolicy: Always
         ports:

manager/Dockerfile

@@ -5,6 +5,7 @@ CMD jar -xf manager-3.0.0.jar BOOT-INF/classes/application.properties && \
     sed -ie "s#^message.broker.host=.*#message.broker.host=$RABBITMQ_HOST#" BOOT-INF/classes/application.properties && \ 
     sed -ie "s#^db.host=.*#db.host=$MONGO_HOST#" BOOT-INF/classes/application.properties && \ 
     sed -ie "s#^sure-tosca.base.path=.*#sure-tosca.base.path=$SURE_TOSCA_BASE_PATH#" BOOT-INF/classes/application.properties && \
+    sed -ie "s#^credential.secret=.*#credential.secret=$CREDENTIAL_SECRET#" BOOT-INF/classes/application.properties && \
     cat BOOT-INF/classes/application.properties && \
     jar -uf manager-3.0.0.jar BOOT-INF/classes/application.properties && \
     java -jar manager-3.0.0.jar

manager/src/main/java/nl/uva/sne/drip/api/CredentialApiController.java

@@ -3,6 +3,9 @@ package nl.uva.sne.drip.api;
 import nl.uva.sne.drip.model.tosca.Credential;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import io.swagger.annotations.*;
+import java.io.UnsupportedEncodingException;
+import java.security.InvalidKeyException;
+import java.security.NoSuchAlgorithmException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.http.HttpStatus;
@@ -12,6 +15,10 @@ import org.springframework.web.bind.annotation.RequestBody;
 import javax.validation.Valid;
 import javax.servlet.http.HttpServletRequest;
 import java.util.List;
+import java.util.logging.Level;
+import javax.crypto.BadPaddingException;
+import javax.crypto.IllegalBlockSizeException;
+import javax.crypto.NoSuchPaddingException;
 import nl.uva.sne.drip.service.CredentialService;
 import org.springframework.beans.factory.annotation.Autowired;
 
@@ -41,8 +48,12 @@ public class CredentialApiController implements CredentialApi {
             @Valid @RequestBody Credential body) {
         String accept = request.getHeader("Accept");
         if (accept != null && accept.contains("application/json")) {
-            String id = credentialService.save(body);
-            return new ResponseEntity<>(id, HttpStatus.OK);
+            try {
+                String id = credentialService.save(body);
+                return new ResponseEntity<>(id, HttpStatus.OK);
+            } catch (UnsupportedEncodingException | NoSuchAlgorithmException | NoSuchPaddingException | InvalidKeyException | IllegalBlockSizeException | BadPaddingException ex) {
+                return new ResponseEntity<>(HttpStatus.INTERNAL_SERVER_ERROR);
+            }
         } else {
             return new ResponseEntity<>(HttpStatus.NOT_ACCEPTABLE);
         }

manager/src/main/java/nl/uva/sne/drip/api/ToscaTemplateApiController.java

@@ -54,7 +54,7 @@ public class ToscaTemplateApiController implements ToscaTemplateApi {
         if (accept != null && accept.contains("text/plain")) {
             try {
                 dripService.delete(id, nodeName);
-                return new ResponseEntity<>("", HttpStatus.OK);
+                return new ResponseEntity<>(id, HttpStatus.OK);
             } catch (IOException | ApiException | TypeExeption | TimeoutException | InterruptedException ex) {
                 java.util.logging.Logger.getLogger(ToscaTemplateApiController.class.getName()).log(Level.SEVERE, null, ex);
                 return new ResponseEntity<>(HttpStatus.INTERNAL_SERVER_ERROR);

manager/src/main/java/nl/uva/sne/drip/service/CredentialService.java

@@ -5,13 +5,22 @@
  */
 package nl.uva.sne.drip.service;
 
-import com.fasterxml.jackson.core.JsonProcessingException;
+import java.io.UnsupportedEncodingException;
+import java.security.InvalidKeyException;
+import java.security.NoSuchAlgorithmException;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import javax.crypto.BadPaddingException;
+import javax.crypto.IllegalBlockSizeException;
+import javax.crypto.NoSuchPaddingException;
+import nl.uva.sne.drip.commons.utils.Converter;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 import nl.uva.sne.drip.dao.CredentialDAO;
 import nl.uva.sne.drip.model.tosca.Credential;
+import org.springframework.beans.factory.annotation.Value;
 
 /**
  *
@@ -20,17 +29,21 @@ import nl.uva.sne.drip.model.tosca.Credential;
 @Service
 public class CredentialService {
 
+    @Value("${credential.secret}")
+    private String credentialSecret;
+
     @Autowired
     private CredentialDAO dao;
 
-    public String save(Credential document) {
-        dao.save(document);
+    public String save(Credential document) throws UnsupportedEncodingException, NoSuchAlgorithmException, NoSuchPaddingException, InvalidKeyException, IllegalBlockSizeException, BadPaddingException {
+
+        dao.save(Converter.encryptCredential(document,credentialSecret));
         return document.getId();
     }
 
-    public Credential findByID(String id) throws JsonProcessingException {
-        Credential credentials = dao.findById(id).get();
-        return credentials;
+    public Credential findByID(String id) {
+        Credential credential = dao.findById(id).get();
+        return credential;
     }
 
     public void deleteByID(String id) {
@@ -54,4 +67,6 @@ public class CredentialService {
         return dao.findBycloudProviderName(provider);
     }
 
+
+
 }

manager/src/main/java/nl/uva/sne/drip/service/DRIPService.java

@@ -67,8 +67,8 @@ public class DRIPService {
         message.setToscaTemplate(toscaTemplate);
 
         caller.setRequestQeueName(requestQeueName);
-        Message plannerResponse = caller.call(message);
-        ToscaTemplate updatedToscaTemplate = plannerResponse.getToscaTemplate();
+        Message response = caller.call(message);
+        ToscaTemplate updatedToscaTemplate = response.getToscaTemplate();
         caller.close();
         return toscaTemplateService.save(updatedToscaTemplate);
 

manager/src/main/resources/application.properties

@@ -18,4 +18,6 @@ db.name=drip
 db.username=drip-user
 db.password=drip-pass
 
-sure-tosca.base.path=http://localhost:8081/tosca-sure/1.0.0
+sure-tosca.base.path=http://localhost:8081/tosca-sure/1.0.0
\ No newline at end of file
+
+credential.secret=top_secret
\ No newline at end of file

manager/src/test/java/nl/uva/sne/drip/service/ServiceTests.java

@@ -28,6 +28,8 @@ import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;
 import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.security.InvalidKeyException;
 import java.security.NoSuchAlgorithmException;
 import java.util.HashMap;
 import java.util.List;
@@ -37,6 +39,9 @@ import java.util.Set;
 import java.util.concurrent.TimeoutException;
 import java.util.logging.Level;
 import java.util.logging.Logger;
+import javax.crypto.BadPaddingException;
+import javax.crypto.IllegalBlockSizeException;
+import javax.crypto.NoSuchPaddingException;
 import nl.uva.sne.drip.Swagger2SpringBoot;
 import nl.uva.sne.drip.api.NotFoundException;
 import nl.uva.sne.drip.commons.utils.Constants;
@@ -301,7 +306,7 @@ public class ServiceTests {
      * Test of save method, of class CredentialService.
      */
     @Test
-    public void testCredentialServiceSave() {
+    public void testCredentialServiceSave() throws UnsupportedEncodingException, NoSuchAlgorithmException, NoSuchPaddingException, InvalidKeyException, IllegalBlockSizeException, BadPaddingException {
         Logger.getLogger(ServiceTests.class.getName()).log(Level.INFO, "save");
         saveCredential();
     }
@@ -332,7 +337,7 @@ public class ServiceTests {
         assertEquals(keystorFileChecksum, keystorFileCopyChecksum);
     }
 
-    public String saveCredential() {
+    public String saveCredential() throws UnsupportedEncodingException, NoSuchAlgorithmException, NoSuchPaddingException, InvalidKeyException, IllegalBlockSizeException, BadPaddingException {
         Logger.getLogger(ServiceTests.class.getName()).log(Level.INFO, "saveCredential");
         Credential document = new Credential();
         document.setCloudProviderName("exogeni");
@@ -364,7 +369,7 @@ public class ServiceTests {
      * @throws com.fasterxml.jackson.core.JsonProcessingException
      */
     @Test
-    public void testCredentialServiceDeleteByID() throws JsonProcessingException {
+    public void testCredentialServiceDeleteByID() throws JsonProcessingException, UnsupportedEncodingException, NoSuchAlgorithmException, NoSuchPaddingException, InvalidKeyException, IllegalBlockSizeException, BadPaddingException {
         Logger.getLogger(ServiceTests.class.getName()).log(Level.INFO, "deleteByID");
         String id = saveCredential();
         credentialService.deleteByID(id);
@@ -382,7 +387,7 @@ public class ServiceTests {
      * Test of getAllIds method, of class CredentialService.
      */
     @Test
-    public void testCredentialServiceGetAllIds() {
+    public void testCredentialServiceGetAllIds() throws UnsupportedEncodingException, NoSuchAlgorithmException, NoSuchPaddingException, InvalidKeyException, IllegalBlockSizeException, BadPaddingException {
         Logger.getLogger(ServiceTests.class.getName()).log(Level.INFO, "getAllIds");
         testCredentialServiceDeleteAll();
         int numOfINst = 3;
@@ -404,7 +409,7 @@ public class ServiceTests {
     }
 
     @Test
-    public void testSetProvisionerOperation() throws FileNotFoundException, IOException, MissingCredentialsException, ApiException, TypeExeption, JsonProcessingException, TimeoutException, InterruptedException, NotFoundException, MissingVMTopologyException {
+    public void testSetProvisionerOperation() throws FileNotFoundException, IOException, MissingCredentialsException, ApiException, TypeExeption, JsonProcessingException, TimeoutException, InterruptedException, NotFoundException, MissingVMTopologyException, UnsupportedEncodingException, NoSuchAlgorithmException, NoSuchPaddingException, InvalidKeyException, IllegalBlockSizeException, BadPaddingException {
         if (ToscaHelper.isServiceUp(sureToscaBasePath) && ToscaHelper.isServiceUp("http://" + messageBrokerHost + ":15672")) {
 
             addRandomCredential("ExoGENI");
@@ -447,7 +452,7 @@ public class ServiceTests {
         }
     }
 
-    private void addRandomCredential(String providerName) {
+    private void addRandomCredential(String providerName) throws UnsupportedEncodingException, NoSuchAlgorithmException, NoSuchPaddingException, InvalidKeyException, IllegalBlockSizeException, BadPaddingException {
         Credential document = new Credential();
         document.setCloudProviderName(providerName);
         Map<String, String> keys = new HashMap<>();

provisioner/Dockerfile

@@ -6,8 +6,14 @@ CMD jar -xf provisioner-3.0.0-jar-with-dependencies.jar application.properties &
     cat application.properties && \
     sed -ie "s#^message.broker.host=.*#message.broker.host=$RABBITMQ_HOST#" application.properties && \ 
     sed -ie "s#^sure-tosca.base.path=.*#sure-tosca.base.path=$SURE_TOSCA_BASE_PATH#" application.properties && \
+    sed -ie "s#^cloud.storm.secret=.*#cloud.storm.secret=$CLOUD_STORM_SECRET#" application.properties && \
+    sed -ie "s#^credential.secret=.*#credential.secret=$CREDENTIAL_SECRET#" application.properties && \
+    sed -ie "s#^cloud.storm.db.path=.*#cloud.storm.db.path=/etc/UD#" application.properties && \
+    echo "" >> application.properties && \
     echo "cloud.storm.db.path=/etc/UD" >> application.properties && \
     cat application.properties && \
     jar -uf provisioner-3.0.0-jar-with-dependencies.jar application.properties && \
     sleep 5 && \
     java -jar provisioner-3.0.0-jar-with-dependencies.jar
+
+    

provisioner/src/main/java/nl/uva/sne/drip/provisioner/CloudStormService.java

@@ -17,9 +17,12 @@ import com.jcraft.jsch.KeyPair;
 import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.IOException;
+import java.io.UnsupportedEncodingException;
 import java.nio.file.Files;
 import java.nio.file.Paths;
 import java.nio.file.attribute.PosixFilePermission;
+import java.security.InvalidKeyException;
+import java.security.NoSuchAlgorithmException;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -29,6 +32,9 @@ import java.util.Properties;
 import java.util.Set;
 import java.util.logging.Level;
 import java.util.logging.Logger;
+import javax.crypto.BadPaddingException;
+import javax.crypto.IllegalBlockSizeException;
+import javax.crypto.NoSuchPaddingException;
 import nl.uva.sne.drip.commons.utils.Constants;
 import static nl.uva.sne.drip.commons.utils.Constants.*;
 import nl.uva.sne.drip.commons.utils.Converter;
@@ -62,6 +68,9 @@ import topology.analysis.TopologyAnalysisMain;
  */
 class CloudStormService {
 
+    protected String secret;
+    private String credentialSecret;
+
     /**
      * @return the helper
      */
@@ -105,6 +114,15 @@ class CloudStormService {
         if (sureToscaBasePath == null) {
             throw new NullPointerException("sureToscaBasePath cannot be null");
         }
+        secret = properties.getProperty("cloud.storm.secret");
+        if (secret == null) {
+            throw new NullPointerException("secret cannot be null");
+        }
+        credentialSecret = properties.getProperty("credential.secret");
+        if (credentialSecret == null) {
+            throw new NullPointerException("secret cannot be null");
+        }
+
         Logger.getLogger(CloudStormService.class.getName()).log(Level.FINE, "sureToscaBasePath: {0}", sureToscaBasePath);
         this.helper = new ToscaHelper(sureToscaBasePath);
         this.helper.uploadToscaTemplate(toscaTemplate);
@@ -123,9 +141,10 @@ class CloudStormService {
         for (NodeTemplateMap vmTopologyMap : helper.getVMTopologyTemplates()) {
             Map<String, Object> provisionedFiles = helper.getNodeArtifact(vmTopologyMap.getNodeTemplate(), "provisioned_files");
             if (provisionedFiles != null) {
-                String fileContentsBase64 = (String) provisionedFiles.get("file_contents");
-                if (fileContentsBase64 != null) {
+                String encryptedFileContents = (String) provisionedFiles.get("file_contents");
+                if (encryptedFileContents != null) {
                     File zipFile = new File(tempInputDir.getParent() + File.separator + Long.toString(System.nanoTime()) + "-" + CLOUD_STORM_FILES_ZIP_SUFIX);
+                    String fileContentsBase64 = Converter.decryptString(encryptedFileContents, secret);
                     Converter.decodeBase64BToFile(fileContentsBase64, zipFile.getAbsolutePath());
                     Converter.unzipFolder(zipFile.getAbsolutePath(), tempInputDir.getAbsolutePath());
 
@@ -282,6 +301,7 @@ class CloudStormService {
         int i = 0;
         for (NodeTemplateMap vmTopologyMap : vmTopologiesMaps) {
             Credential toscaCredentials = getHelper().getCredentialsFromVMTopology(vmTopologyMap);
+            toscaCredentials = Converter.dencryptCredential(toscaCredentials, credentialSecret);
             CloudCred cloudStormCredential = new CloudCred();
             cloudStormCredential.setCloudProvider(toscaCredentials.getCloudProviderName());
             String credInfoFile = toscaCredentials.getCloudProviderName() + i + ".yml";
@@ -355,7 +375,7 @@ class CloudStormService {
         FileUtils.copyDirectory(srcDir, destDir);
     }
 
-    protected ToscaTemplate runCloudStorm(String tempInputDirPath, boolean dryRun) throws IOException, ApiException {
+    protected ToscaTemplate runCloudStorm(String tempInputDirPath, boolean dryRun) throws IOException, ApiException, UnsupportedEncodingException, NoSuchAlgorithmException, NoSuchPaddingException, InvalidKeyException, IllegalBlockSizeException, BadPaddingException {
         String[] args = new String[]{"run", tempInputDirPath};
         File topTopologyFile = new File(tempInputDirPath + TOPOLOGY_RELATIVE_PATH
                 + TOP_TOPOLOGY_FILE_NAME);
@@ -402,7 +422,7 @@ class CloudStormService {
         return keyPair;
     }
 
-    protected NodeTemplateMap addCloudStromArtifacts(NodeTemplateMap vmTopologyMap, String tempInputDirPath) throws IOException {
+    protected NodeTemplateMap addCloudStromArtifacts(NodeTemplateMap vmTopologyMap, String tempInputDirPath) throws IOException, UnsupportedEncodingException, NoSuchAlgorithmException, NoSuchPaddingException, InvalidKeyException, IllegalBlockSizeException, BadPaddingException {
         Map<String, Object> artifacts = vmTopologyMap.getNodeTemplate().getArtifacts();
         if (artifacts == null) {
             artifacts = new HashMap<>();
@@ -416,7 +436,8 @@ class CloudStormService {
         Logger.getLogger(CloudStormService.class.getName()).log(Level.FINE, "Created zip at: {0}", zipPath);
 
         String cloudStormZipFileContentsAsBase64 = Converter.encodeFileToBase64Binary(zipPath);
-        provisionedFiles.put("file_contents", cloudStormZipFileContentsAsBase64);
+        String encryptedCloudStormZipFileContents = Converter.encryptString(cloudStormZipFileContentsAsBase64, secret);
+        provisionedFiles.put("file_contents", encryptedCloudStormZipFileContents);
         provisionedFiles.put("encoding", "base64");
         provisionedFiles.put("file_ext", "zip");
         artifacts.put("provisioned_files", provisionedFiles);
@@ -425,7 +446,7 @@ class CloudStormService {
         return vmTopologyMap;
     }
 
-    private void setSSHKeysToVMAttributes(int i, List<NodeTemplateMap> vmTopologiesMaps, CloudsStormSubTopology subTopology, String tempInputDirPath) throws IOException, ApiException {
+    private void setSSHKeysToVMAttributes(int i, List<NodeTemplateMap> vmTopologiesMaps, CloudsStormSubTopology subTopology, String tempInputDirPath) throws IOException, ApiException, UnsupportedEncodingException, NoSuchAlgorithmException, NoSuchPaddingException, InvalidKeyException, IllegalBlockSizeException, BadPaddingException {
         NodeTemplateMap vmTopologyMap = vmTopologiesMaps.get(i);
 
         vmTopologyMap = addCloudStromArtifacts(vmTopologyMap, tempInputDirPath);
@@ -459,7 +480,7 @@ class CloudStormService {
         userKyes.put("private_key", Converter.encodeFileToBase64Binary(userKyePairFolder + File.separator + "id_rsa"));
         userKyes.put("public_key", Converter.encodeFileToBase64Binary(userKyePairFolder + File.separator + "id_rsa.pub"));
         userKeyPairCredential.setKeys(userKyes);
-
+//        userKeyPairCredential = Converter.encryptCredential(userKeyPairCredential, credentialSecret);
         CloudsStormVMs cloudsStormVMs = objectMapper.readValue(new File(tempInputDirPath + TOPOLOGY_RELATIVE_PATH + File.separator + subTopology.getTopology() + ".yml"),
                 CloudsStormVMs.class);
         List<CloudsStormVM> vms = cloudsStormVMs.getVms();

provisioner/src/main/resources/application.properties

@@ -5,4 +5,6 @@ message.broker.queue.provisioner=provisioner
 message.broker.queue.planner=planner
 message.broker.queue.deployer=deployer
 sure-tosca.base.path=http://localhost:8081/tosca-sure/1.0.0
+cloud.storm.secret=top_secret
+credential.secret=top_secret
 cloud.storm.db.path=etc/UD
\ No newline at end of file

provisioner/src/test/java/nl/uva/sne/drip/provisioner/CloudStormServiceTest.java

@@ -284,6 +284,7 @@ public class CloudStormServiceTest {
 
                 assertTrue(provisionedFiles.containsKey("file_contents"));
                 String cloudStormZipFileContentsAsBase64 = provisionedFiles.get("file_contents");
+                cloudStormZipFileContentsAsBase64 = Converter.decryptString(cloudStormZipFileContentsAsBase64, instance.secret);
                 String testZipPath = tempInputDirPath + File.separator + "test.zip";
 
                 Converter.decodeBase64BToFile(cloudStormZipFileContentsAsBase64, testZipPath);

provisioner/src/test/resources/application.properies

-message.broker.host=127.0.0.1
-message.broker.username=guest
-message.broker.password=guest
-message.broker.queue.provisioner=provisioner
-message.broker.queue.planner=planner
-message.broker.queue.deployer=deployer
-sure-tosca.base.path=http://localhost:8081/tosca-sure/1.0.0

semaphore-python-client-generated/semaphore_client/semaphore_helper.py

+import json
 from datetime import datetime
 
 import urllib3
 
 from semaphore_client import Configuration, ApiClient, api, ProjectRequest, Login, AccessKeyRequest, InventoryRequest, \
-    RepositoryRequest, TemplateRequest, Task
+    RepositoryRequest, TemplateRequest, Task, EnvironmentRequest
 
 
 class SemaphoreHelper:
@@ -77,8 +78,12 @@ class SemaphoreHelper:
         templates = self.project_api.project_project_id_templates_get(project_id, playbook_name, 'asc')
         return self.find_template(templates,playbook_name).id
 
-    def execute_task(self, project_id, template_id, playbook_name):
-        task = Task(template_id=template_id, playbook=playbook_name)
+    def execute_task(self, project_id, template_id, playbook_name,environment_id=None):
+        environment_json = None
+        if environment_id:
+            environment = self.find_environment(project_id,environment_id=environment_id)
+            environment_json = environment.json
+        task = Task(template_id=template_id, playbook=playbook_name,environment=environment_json)
         self.project_api.project_project_id_tasks_post(task,project_id)
         tasks = self.project_api.project_project_id_tasks_get(project_id)
         return self.find_latest_task(tasks).id
@@ -126,4 +131,21 @@ class SemaphoreHelper:
         for repo in repositories:
             if repo.git_url == git_url:
                 return repo
+        return None
+
+    def create_environment(self, project_id,name,env_vars):
+        vars = {"ENV":env_vars}
+        vars_str = json.dumps(vars)
+        environment_request = EnvironmentRequest(name=name, project_id=project_id, password=None, json=vars_str)
+        self.project_api.project_project_id_environment_post(environment_request,project_id)
+        environment = self.find_environment(project_id, name,vars_str)
+        return environment.id
+
+    def find_environment(self, project_id, name=None,vars_str=None,environment_id=None):
+        environments = self.project_api.project_project_id_environment_get(project_id,'db-deploy','desc')
+        for environment in environments:
+            if environment_id and environment.id ==environment_id:
+                return environment
+            if environment.name and environment.name == name and environment.project_id == project_id and environment.json == vars_str:
+                return environment
         return None
\ No newline at end of file

semaphore-python-client-generated/test/test_task.py

@@ -15,7 +15,7 @@ from __future__ import absolute_import
 
 import unittest
 from time import sleep
-
+import datetime
 import yaml
 
 from semaphore_client.semaphore_helper import SemaphoreHelper
@@ -30,22 +30,23 @@ class TestTask(unittest.TestCase):
         if SemaphoreHelper.service_is_up(self.semaphore_base_url):
             self.username = 'admin'
             self.password = 'password'
-            self.project_name = 'test'
+            now = datetime.datetime.now()
+            self.project_name = 'project_name_test'+'_'+str(now)
             self.private_key = '-----BEGIN RSA PRIVATE KEY-----MIIEowIBAAKCAQEAg0blRNV6cm3RTiivpzE8HR4JzKZRVIBZ7bxeNoMz0' \
                                '-----END RSA PRIVATE KEY-----'
             vms = []
             vm = {}
             vm['role'] = 'master'
-            vm['public_ip'] = '192.168.1.10'
-            vm['user_name'] = 'user_name'
+            vm['public_ip'] = '127.0.0.1'
+            vm['semaphore'] = 'semaphore'
             vms.append(vm)
             vm['role'] = 'worker'
-            vm['public_ip'] = '192.168.1.11'
+            vm['public_ip'] = '127.0.0.1'
             vm['role'] = 'worker'
-            vm['public_ip'] = '192.168.1.12'
+            vm['public_ip'] = '127.0.0.1'
             vms.append(vm)
             self.inventory_contents =  yaml.dump( self.build_yml_inventory(vms),default_flow_style=False)
-            self.git_url = 'https://github.com/skoulouzis/playbooks.git'
+            self.git_url = 'https://github.com/QCDIS/playbooks.git'
             self.playbook_name = 'get_ip_addresses.yaml'
             self.semaphore_helper = SemaphoreHelper(self.semaphore_base_url, self.username, self.password)
 
@@ -55,12 +56,22 @@ class TestTask(unittest.TestCase):
     def testTask(self):
         if SemaphoreHelper.service_is_up(self.semaphore_base_url):
             project_id = self.semaphore_helper.create_project(self.project_name)
-            key_id = self.semaphore_helper.create_ssh_key(self.project_name, project_id, self.private_key)
-            inventory_id = self.semaphore_helper.create_inventory(self.project_name, project_id, key_id, self.inventory_contents)
-
-            repository_id = self.semaphore_helper.create_repository(self.project_name, project_id, key_id, self.git_url)
+            now = datetime.datetime.now()
+            key_name = 'key_name'+str(now)
+
+            key_id = self.semaphore_helper.create_ssh_key(key_name, project_id, self.private_key)
+            now = datetime.datetime.now()
+            inventory_name = 'inventory_name'+str(now)
+            inventory_id = self.semaphore_helper.create_inventory(inventory_name, project_id, key_id, self.inventory_contents)
+            now = datetime.datetime.now()
+            repository_name = 'repository_name'+str(now)
+            repository_id = self.semaphore_helper.create_repository(repository_name, project_id, key_id, self.git_url)
             template_id = self.semaphore_helper.create_template(project_id, key_id, inventory_id, repository_id, self.playbook_name)
-            task_id = self.semaphore_helper.execute_task(project_id, template_id, self.playbook_name)
+            env_vars = {"SOME_VAR": "the contents"}
+            now = datetime.datetime.now()
+            environment_name = 'environment_name_name_'+str(now)
+            environment_id = self.semaphore_helper.create_environment(project_id,environment_name,env_vars)
+            task_id = self.semaphore_helper.execute_task(project_id, template_id, self.playbook_name,environment_id=environment_id)
             for x in range(0, 2):
                 task = self.semaphore_helper.get_task(project_id, task_id)
                 print(task)
@@ -68,20 +79,20 @@ class TestTask(unittest.TestCase):
                 # print(task_output)
                 sleep(1)
 
-            task_id = self.semaphore_helper.execute_task(project_id, template_id, self.playbook_name)
-            for x in range(0, 2):
-                task = self.semaphore_helper.get_task(project_id, task_id)
-                print(task)
-                # task_output = self.semaphore_helper.get_task_output(project_id, task_id)
-                # print(task_output)
-                sleep(1)
-            task_id = self.semaphore_helper.execute_task(project_id, template_id, 'mount.yaml')
-            for x in range(0, 2):
-                task = self.semaphore_helper.get_task(project_id, task_id)
-                print(task)
-                # task_output = self.semaphore_helper.get_task_output(project_id, task_id)
-                # print(task_output)
-                sleep(1)
+            # task_id = self.semaphore_helper.execute_task(project_id, template_id, self.playbook_name)
+            # for x in range(0, 2):
+            #     task = self.semaphore_helper.get_task(project_id, task_id)
+            #     print(task)
+            #     # task_output = self.semaphore_helper.get_task_output(project_id, task_id)
+            #     # print(task_output)
+            #     sleep(1)
+            # task_id = self.semaphore_helper.execute_task(project_id, template_id, 'get_ip_addresses.yaml')
+            # for x in range(0, 2):
+            #     task = self.semaphore_helper.get_task(project_id, task_id)
+            #     print(task)
+            #     # task_output = self.semaphore_helper.get_task_output(project_id, task_id)
+            #     # print(task_output)
+            #     sleep(1)
 
 
 
@@ -89,7 +100,7 @@ class TestTask(unittest.TestCase):
         inventory = {}
         all = {}
         vars = {'ansible_ssh_common_args':'-o StrictHostKeyChecking=no'}
-        vars['ansible_ssh_user'] = vms[0]['user_name']
+        vars['ansible_ssh_user'] = vms[0]['semaphore']
         children = {}
         for vm in vms:
             role = vm['role']

sure_tosca-client_python_stubs/test/test_default_api.py

@@ -14,9 +14,10 @@ from __future__ import absolute_import
 
 import os
 import unittest
+from urllib.error import HTTPError
 
-import urllib3
-
+from urllib.request import urlopen
+import ssl
 import sure_tosca_client
 from sure_tosca_client import Configuration, ApiClient
 from sure_tosca_client.api.default_api import DefaultApi  # noqa: E501
@@ -28,7 +29,7 @@ class TestDefaultApi(unittest.TestCase):
     def setUp(self):
         configuration = Configuration()
         configuration.host = 'http://127.0.0.1:8081/tosca-sure/1.0.0' #Make sure we don't have '/' on the end of url
-        if self.service_is_up(configuration.host):
+        if self.check_service_is_up(configuration.host):
             configuration.verify_ssl = False
             api_client = ApiClient(configuration=configuration)
             self.api = sure_tosca_client.api.default_api.DefaultApi(api_client=api_client)  # noqa: E501
@@ -190,12 +191,17 @@ class TestDefaultApi(unittest.TestCase):
             return file_id
 
 
-    def service_is_up(self, url):
+    def check_service_is_up(self, url):
         code = None
         try:
-            http = urllib3.PoolManager()
-            r = http.request('HEAD', url)
+            response = urlopen(url, context=ssl._create_unverified_context())
+            print(response)
+            # http = urllib3.PoolManager()
+            # r = http.request('HEAD', url,context=ctx)
         except Exception as e:
+            if isinstance(e, HTTPError) and e.code == 401:
+                return True
+
             return False
 
         return True

sure_tosca-flask-server/.dockerignore

@@ -3,7 +3,7 @@
 README.md
 tox.ini
 git_push.sh
-test-requirements.txt
+#test-requirements.txt
 setup.py
 
 # Byte-compiled / optimized / DLL files

sure_tosca-flask-server/Dockerfile

-FROM python:3.7-buster
+FROM python:3.6-buster
 
 RUN mkdir -p /usr/src/app
 WORKDIR /usr/src/app
 
 COPY requirements.txt /usr/src/app/
+COPY test-requirements.txt /usr/src/app/
+
 
 RUN pip install --no-cache-dir -r requirements.txt
+#RUN pip install --no-cache-dir -r test-requirements.txt
 
 COPY . /usr/src/app
 
+#RUN python -m unittest discover
+#RUN pip uninstall -y --no-cache-dir -r test-requirements.txt
+
+
 EXPOSE 8081
 
 ENTRYPOINT ["python"]

sure_tosca-flask-server/requirements.txt

@@ -5,4 +5,5 @@ typing==3.7.4.1
 swagger-ui-bundle==0.0.6
 tinydb==3.15.2
 tosca-parser==2.0.0
-flask-restx==0.2.0
+flask-restx==0.2.0
\ No newline at end of file
+werkzeug==1.0.1
\ No newline at end of file

sure_tosca-flask-server/sure_tosca/service/tosca_template_service.py

-import json
+import copy
 import logging
 import os
 import tempfile
 import time
-import uuid
-from builtins import print
 from functools import reduce
-import copy
 
 import yaml
 from tinydb import TinyDB, Query
 from tinydb.middlewares import CachingMiddleware
 from tinydb.storages import MemoryStorage
-from toscaparser.functions import GetAttribute
 from toscaparser.tosca_template import ToscaTemplate
 from werkzeug.datastructures import FileStorage
 
@@ -20,7 +16,6 @@ from sure_tosca.models.base_model_ import Model
 from sure_tosca.models.node_template import NodeTemplateModel as NodeTemplateModel
 from sure_tosca.models.node_template_map import NodeTemplateMapModel
 from sure_tosca.models.tosca_template import ToscaTemplateModel  as ToscaTemplateModel
-
 from sure_tosca.service import tosca_helper
 
 # db = TinyDB(storage=CachingMiddleware(MemoryStorage))
@@ -110,7 +105,7 @@ def save(file: FileStorage):
     purge_all_tables()
     dictionary = yaml.safe_load(file.stream)
     # dictionary = yaml.load(file.stream)
-    logger.info("tosca template: \n" + str(yaml.dump(dictionary)))
+    logger.debug("tosca template: \n" + str(yaml.dump(dictionary)))
     # print(yaml.dump(dictionary))
     tosca_template = get_tosca_template(dictionary)
     # all_custom_def = tosca_template.nodetemplates[0].custom_def

sure_tosca-flask-server/sure_tosca/service/tosca_type_service.py

-from sure_tosca.models.node_template import NodeTemplateModel  # noqa: E501
-from sure_tosca.models.node_template_map import NodeTemplateMapModel  # noqa: E501
-from sure_tosca.models.tosca_template import ToscaTemplateModel  # noqa: E501
-
 def add_type_definition(definition_map):
     return None
 

sure_tosca-flask-server/sure_tosca/test/test_default_controller.py

@@ -2,25 +2,59 @@
 
 from __future__ import absolute_import
 
+import os
+import tempfile
+
+import requests
 from flask import json
 from six import BytesIO
-import os
 
-from sure_tosca.models.node_template import NodeTemplateModel  # noqa: E501
-from sure_tosca.models.topology_template import TopologyTemplateModel  # noqa: E501
-from sure_tosca.models.tosca_template import ToscaTemplateModel  # noqa: E501
 from sure_tosca.test import BaseTestCase
 
 
 class TestDefaultController(BaseTestCase):
     """DefaultController integration test stubs"""
 
+
+    def test_upload(self):
+        """Test case for get_all_ancestor_properties
+
+
+        """
+        id_example = self.upload_file('https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/TIC.yaml')
+        doc_id = int(id_example)
+        self.assertIsInstance(doc_id,int)
+        # id_example = self.upload_file('https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/application_example_planed.yaml')
+        # doc_id = int(id_example)
+        self.assertIsInstance(doc_id,int)
+        id_example = self.upload_file(
+            'https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/application_example_provisioned.yaml')
+        doc_id = int(id_example)
+        self.assertIsInstance(doc_id,int)
+        id_example = self.upload_file(
+            'https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/glusterFS.yaml')
+        doc_id = int(id_example)
+        self.assertIsInstance(doc_id,int)
+        id_example = self.upload_file(
+            'https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/kubernetes.yaml')
+        doc_id = int(id_example)
+        self.assertIsInstance(doc_id,int)
+        id_example = self.upload_file(
+            'https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/lifeWatch_vre1.yaml')
+        doc_id = int(id_example)
+        self.assertIsInstance(doc_id,int)
+        id_example = self.upload_file(
+            'https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/topology.yaml')
+        doc_id = int(id_example)
+        self.assertIsInstance(doc_id,int)
+
+
     def test_get_all_ancestor_properties(self):
         """Test case for get_all_ancestor_properties
 
 
         """
-        id_example = self.upload_file('application_example_2_topologies.yaml')
+        id_example = self.upload_file('https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/application_example_2_topologies.yaml')
         self.assertTrue(id_example.strip().isdigit())
         response = self.client.open(
             '/tosca-sure/1.0.0/tosca_template/{id}/topology_template/node_templates/{node_name}/ancestors_properties'.format(
@@ -35,7 +69,7 @@ class TestDefaultController(BaseTestCase):
 
 
         """
-        id_example = self.upload_file('application_example_2_topologies.yaml')
+        id_example = self.upload_file('https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/application_example_2_topologies.yaml')
         self.assertTrue(id_example.strip().isdigit())
         response = self.client.open(
             '/tosca-sure/1.0.0/tosca_template/{id}/topology_template/node_templates/{node_name}/ancestors_types'.format(
@@ -50,7 +84,7 @@ class TestDefaultController(BaseTestCase):
 
 
         """
-        id_example = self.upload_file('application_example_2_topologies.yaml')
+        id_example = self.upload_file('https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/application_example_2_topologies.yaml')
         self.assertTrue(id_example.strip().isdigit())
         response = self.client.open(
             '/tosca-sure/1.0.0/tosca_template/{id}/topology_template/node_templates/{node_name}/ancestors_requirements'.format(
@@ -66,7 +100,7 @@ class TestDefaultController(BaseTestCase):
 
         """
         # query_string = [('anchors', 'anchors_example'), ('derived_from', 'derived_from_example')]
-        id_example = self.upload_file('application_example_2_topologies.yaml')
+        id_example = self.upload_file('https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/application_example_2_topologies.yaml')
         self.assertTrue(id_example.strip().isdigit())
         response = self.client.open(
             '/tosca-sure/1.0.0/tosca_template/{id}/dsl_definitions'.format(id=id_example),
@@ -78,7 +112,7 @@ class TestDefaultController(BaseTestCase):
 
 
         """
-        id_example = self.upload_file('application_example_2_topologies.yaml')
+        id_example = self.upload_file('https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/application_example_2_topologies.yaml')
         self.assertTrue(id_example.strip().isdigit())
         response = self.client.open(
             '/tosca-sure/1.0.0/tosca_template/{id}/imports'.format(id=id_example),
@@ -90,7 +124,7 @@ class TestDefaultController(BaseTestCase):
 
 
         """
-        id_example = self.upload_file('application_example_outputs.yaml')
+        id_example = self.upload_file('https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/application_example_outputs.yaml')
         self.assertTrue(id_example.strip().isdigit())
         response = self.client.open(
             '/tosca-sure/1.0.0/tosca_template/{id}/topology_template/node_templates/{node_name}/outputs'.format(
@@ -105,7 +139,7 @@ class TestDefaultController(BaseTestCase):
 
 
         """
-        id_example = self.upload_file('application_example_2_topologies.yaml')
+        id_example = self.upload_file('https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/application_example_2_topologies.yaml')
         self.assertTrue(id_example.strip().isdigit())
         response = self.client.open(
             '/tosca-sure/1.0.0/tosca_template/{id}/topology_template/node_templates/{node_name}/properties'.format(
@@ -120,7 +154,7 @@ class TestDefaultController(BaseTestCase):
 
 
         """
-        id_example = self.upload_file('application_example_2_topologies.yaml')
+        id_example = self.upload_file('https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/application_example_2_topologies.yaml')
         self.assertTrue(id_example.strip().isdigit())
         response = self.client.open(
             '/tosca-sure/1.0.0/tosca_template/{id}/topology_template/node_templates/{node_name}/requirements'.format(
@@ -135,7 +169,7 @@ class TestDefaultController(BaseTestCase):
 
 
         """
-        id_example = self.upload_file('application_example_2_topologies.yaml')
+        id_example = self.upload_file('https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/application_example_2_topologies.yaml')
         self.assertTrue(id_example.strip().isdigit())
         query_string = [('type_name', None),
                         ('node_name', 'compute'),
@@ -206,7 +240,7 @@ class TestDefaultController(BaseTestCase):
 
 
         """
-        id_example = self.upload_file('application_example_2_topologies.yaml')
+        id_example = self.upload_file('https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/application_example_2_topologies.yaml')
         self.assertTrue(id_example.strip().isdigit())
         response = self.client.open(
             '/tosca-sure/1.0.0/tosca_template/{id}/topology_template/node_templates/{node_name}/type_name'.format(
@@ -221,7 +255,7 @@ class TestDefaultController(BaseTestCase):
 
 
         """
-        id_example = self.upload_file('application_example_2_topologies.yaml')
+        id_example = self.upload_file('https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/application_example_2_topologies.yaml')
         self.assertTrue(id_example.strip().isdigit())
         response = self.client.open(
             '/tosca-sure/1.0.0/tosca_template/{id}/topology_template/node_templates/{node_name}/derived_from'.format(
@@ -236,7 +270,7 @@ class TestDefaultController(BaseTestCase):
 
 
         """
-        id_example = self.upload_file('application_example_2_topologies.yaml')
+        id_example = self.upload_file('https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/application_example_2_topologies.yaml')
         self.assertTrue(id_example.strip().isdigit())
         response = self.client.open(
             '/tosca-sure/1.0.0/tosca_template/{id}/topology_template/node_templates/{node_name}/related'.format(
@@ -251,7 +285,7 @@ class TestDefaultController(BaseTestCase):
 
 
         """
-        id_example = self.upload_file('application_example_2_topologies.yaml')
+        id_example = self.upload_file('https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/application_example_2_topologies.yaml')
         self.assertTrue(id_example.strip().isdigit())
         query_string = [('type_name', None),
                         ('derived_from', None)]
@@ -266,7 +300,7 @@ class TestDefaultController(BaseTestCase):
 
 
         """
-        id_example = self.upload_file('application_example_2_topologies.yaml')
+        id_example = self.upload_file('https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/application_example_2_topologies.yaml')
         self.assertTrue(id_example.strip().isdigit())
         response = self.client.open(
             '/tosca-sure/1.0.0/tosca_template/{id}/topology_template'.format(id=id_example),
@@ -280,7 +314,7 @@ class TestDefaultController(BaseTestCase):
 
 
         """
-        id_example = self.upload_file('application_example_2_topologies.yaml')
+        id_example = self.upload_file('https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/application_example_2_topologies.yaml')
         self.assertTrue(id_example.strip().isdigit())
         response = self.client.open(
             '/tosca-sure/1.0.0/tosca_template/{id}'.format(id=id_example),
@@ -294,7 +328,7 @@ class TestDefaultController(BaseTestCase):
 
 
         """
-        id_example = self.upload_file('application_example_2_topologies.yaml')
+        id_example = self.upload_file('https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/application_example_2_topologies.yaml')
         self.assertTrue(id_example.strip().isdigit())
         query_string = [('kind_of_type', 'interface_types'),
                         ('has_interfaces', None),
@@ -334,7 +368,7 @@ class TestDefaultController(BaseTestCase):
 
 
         """
-        id_example = self.upload_file('application_example_2_topologies.yaml')
+        id_example = self.upload_file('https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/application_example_2_topologies.yaml')
         self.assertTrue(id_example.strip().isdigit())
         properties = {'properties': {'cpu_frequency': '2 GHz'}}
         response = self.client.open(
@@ -354,7 +388,7 @@ class TestDefaultController(BaseTestCase):
 
 
         """
-        id_example = self.upload_file('application_example_2_topologies.yaml')
+        id_example = self.upload_file('https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/application_example_2_topologies.yaml')
         self.assertTrue(id_example.strip().isdigit())
         response = self.client.open(
             '/tosca-sure/1.0.0/tosca_template/{id}/topology_template/node_templates'.format(id=id_example),
@@ -370,7 +404,7 @@ class TestDefaultController(BaseTestCase):
 
 
         """
-        id_example = self.upload_file('application_example_2_topologies.yaml')
+        id_example = self.upload_file('https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/application_example_2_topologies.yaml')
         self.assertTrue(id_example.strip().isdigit())
         query_string = [('instance_name', 'instance_name_example'),
                         ('operation_name', 'provision')]
@@ -383,13 +417,8 @@ class TestDefaultController(BaseTestCase):
         self.assertEqual(response.status_code, 200)
         self.assertIsInstance(response.json, dict)
 
-    def upload_file(self, file_name):
-        tosca_path = "../../../TOSCA/"
-        input_tosca_file_path = tosca_path + '/' + file_name
-        if not os.path.exists(input_tosca_file_path):
-            tosca_path = "../TOSCA/"
-            input_tosca_file_path = tosca_path + '/' + file_name
-
+    def upload_file(self, url):
+        input_tosca_file_path = self.get_remote_tosca_file(url)
         dir_path = os.path.dirname(os.path.realpath(__file__))
         self.assertEqual(True, os.path.exists(input_tosca_file_path),
                          'Starting from: ' + dir_path + ' Input TOSCA file: ' + input_tosca_file_path + ' not found')
@@ -405,6 +434,11 @@ class TestDefaultController(BaseTestCase):
             content_type='multipart/form-data')
         return response.data.decode('utf-8').replace('\n', '')
 
+    def get_remote_tosca_file(self, url):
+        tosca = requests.get(url)
+        input_tosca_file_path = os.path.join(tempfile.gettempdir(),'test_tosca_file.yaml')
+        open( input_tosca_file_path, 'wb').write(tosca.content)
+        return input_tosca_file_path
 
 if __name__ == '__main__':
     import unittest

sure_tosca-flask-server/sure_tosca/test/test_tosca_template_service.py

 import os
+import tempfile
 from unittest import TestCase
+
+import requests
 from six import BytesIO
 from werkzeug.datastructures import FileStorage
 import logging
@@ -15,16 +18,18 @@ formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(messag
 h.setFormatter(formatter)
 logger.addHandler(h)
 logger.handler_set = True
+base_url = 'https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/'
 
 class Test(TestCase):
     def test_upload(self):
+
         file_names = ['application_example_outputs.yaml','application_example_2_topologies.yaml','lifeWatch_vre1.yaml',
                       'application_example_updated.yaml','compute.yaml','application_example_provisioned.yaml','topology.yaml',
-                      'kubernetes.yaml','application_example_planed.yaml']
+                      'kubernetes.yaml','TIC.yaml']
 
         for file_name in file_names:
             logger.info("Testing : " + str(file_name))
-            doc_id = tosca_template_service.save(self.upload_file(file_name))
+            doc_id = tosca_template_service.save(self.upload_file(base_url+file_name))
             logger.info("doc_id : " + str(doc_id))
             self.assertTrue (doc_id >= 0)
 
@@ -39,7 +44,7 @@ class Test(TestCase):
 
 
     def test_get_tosca_template_model_by_id(self):
-        doc_id = tosca_template_service.save(self.upload_file('application_example_updated.yaml'))
+        doc_id = tosca_template_service.save(self.upload_file(base_url+'application_example_updated.yaml'))
         tosca_template_dict = tosca_template_service.get_tosca_template_dict_by_id(doc_id)
         tosca_template_model = ToscaTemplateModel.from_dict(tosca_template_dict)
 
@@ -47,17 +52,12 @@ class Test(TestCase):
         self.assertIsNotNone(tosca_template_model.topology_template)
         self.assertIsNotNone(tosca_template_model.topology_template.node_templates)
 
-    def upload_file(self, file_name):
-        tosca_path = "../../../TOSCA/"
-        input_tosca_file_path = tosca_path + '/' + file_name
-        if not os.path.exists(input_tosca_file_path):
-            tosca_path = "../TOSCA/"
-            input_tosca_file_path = tosca_path + '/' + file_name
 
+    def upload_file(self, url):
+        input_tosca_file_path = self.get_remote_tosca_file(url)
         dir_path = os.path.dirname(os.path.realpath(__file__))
         self.assertEqual(True, os.path.exists(input_tosca_file_path),
                          'Starting from: ' + dir_path + ' Input TOSCA file: ' + input_tosca_file_path + ' not found')
-
         file = open(input_tosca_file_path, "r")
         # with open(input_tosca_file_path, 'r') as file:
         #     contents = file.read()
@@ -71,3 +71,10 @@ class Test(TestCase):
         return FileStorage(stream=BytesIO(byte_contents), filename=input_tosca_file_path, name=input_tosca_file_path,
                            content_type=None,
                            content_length=None, headers=None)
+
+
+    def get_remote_tosca_file(self, url):
+        tosca = requests.get(url)
+        input_tosca_file_path = os.path.join(tempfile.gettempdir(),'test_tosca_file.yaml')
+        open( input_tosca_file_path, 'wb').write(tosca.content)
+        return input_tosca_file_path