fix error with indentation: we where not storing the outputs of some tasks

deployer/__main__.py

@@ -8,14 +8,16 @@ import os
 import sys
 import tempfile
 import time
+import traceback
 from threading import Thread
 from time import sleep
-from service.deploy_service import DeployService
-from service.tosca_helper import  ToscaHelper
+
 import pika
-import sure_tosca_client
 import yaml
 
+from service.deploy_service import DeployService
+from service.tosca_helper import ToscaHelper
+
 logger = logging.getLogger(__name__)
 
 done = False
@@ -85,9 +87,21 @@ def handle_delivery(message):
     nodes_pairs = tosca_helper.get_deployment_node_pairs()
 
     deployService = DeployService(semaphore_base_url=semaphore_base_url, semaphore_username=semaphore_username,
-                                  semaphore_password=semaphore_password,vms=tosca_helper.get_vms())
-    for node_pair in nodes_pairs:
-        deployService.deploy(node_pair)
+                                  semaphore_password=semaphore_password, vms=tosca_helper.get_vms())
+    try:
+        for node_pair in nodes_pairs:
+            updated_node_pairs = deployService.deploy(node_pair)
+            for updated_node in updated_node_pairs:
+                if isinstance(updated_node, list):
+                    for node in updated_node:
+                        tosca_helper.set_node(node,tosca_template_dict)
+                else:
+                    tosca_helper.set_node(updated_node, tosca_template_dict)
+
+    except Exception as e:
+        track = traceback.format_exc()
+        print(track)
+        raise
 
     response = {'toscaTemplate': tosca_template_dict}
     output_current_milli_time = int(round(time.time() * 1000))

deployer/service/ansible_service.py

@@ -6,7 +6,7 @@ import datetime
 import yaml
 from semaphore_client.semaphore_helper import SemaphoreHelper
 
-yaml.Dumper.ignore_aliases = lambda *args : True
+yaml.Dumper.ignore_aliases = lambda *args: True
 
 logger = logging.getLogger(__name__)
 if not getattr(logger, 'handler_set', None):
@@ -20,16 +20,15 @@ if not getattr(logger, 'handler_set', None):
 
 class AnsibleService:
 
-    def __init__(self, semaphore_base_url=None,semaphore_username=None,semaphore_password=None):
+    def __init__(self, semaphore_base_url=None, semaphore_username=None, semaphore_password=None):
         self.semaphore_base_url = semaphore_base_url
         self.semaphore_username = semaphore_username
         self.semaphore_password = semaphore_password
-        self.semaphore_helper = SemaphoreHelper(self.semaphore_base_url, self.semaphore_username, self.semaphore_password)
+        self.semaphore_helper = SemaphoreHelper(self.semaphore_base_url, self.semaphore_username,
+                                                self.semaphore_password)
         self.repository_id = None
         self.template_id = None
 
-
-
     def execute(self, nodes_pair, interface_type, vms, env_vars=None):
         application = nodes_pair[1]
         name = application.name
@@ -43,11 +42,12 @@ class AnsibleService:
 
         if desired_state:
             now = datetime.datetime.now()
-            project_id = self.semaphore_helper.create_project(application.name+'_'+str(now))
-            inventory_contents = yaml.dump( self.build_yml_inventory(vms),default_flow_style=False)
+            project_id = self.semaphore_helper.create_project(application.name + '_' + str(now))
+            inventory_contents = yaml.dump(self.build_yml_inventory(vms), default_flow_style=False)
             private_key = self.get_private_key(vms)
             key_id = self.semaphore_helper.create_ssh_key(application.name, project_id, private_key)
-            inventory_id = self.semaphore_helper.create_inventory(application.name, project_id, key_id,inventory_contents)
+            inventory_id = self.semaphore_helper.create_inventory(application.name, project_id, key_id,
+                                                                  inventory_contents)
             if 'RUNNING' == desired_state:
                 interface = interfaces[interface_type]
                 create = interface['create']
@@ -57,13 +57,17 @@ class AnsibleService:
                 for playbook_name in playbook_names:
                     environment_id = None
                     if env_vars:
-                        environment_id = self.semaphore_helper.create_environment(project_id,name,env_vars)
-                    task_id = self.run_task(name, project_id, key_id, git_url, inventory_id, playbook_name, environment_id= environment_id)
+                        environment_id = self.semaphore_helper.create_environment(project_id, name, env_vars)
+                    task_id = self.run_task(name, project_id, key_id, git_url, inventory_id, playbook_name,
+                                            environment_id=environment_id)
                     if self.semaphore_helper.get_task(project_id, task_id).status != 'success':
                         break
+                    logger.info('playbook: ' + playbook_name + ' task_id: ' + str(task_id))
                     tasks_outputs[task_id] = self.semaphore_helper.get_task_outputs(project_id, task_id)
+                    for out in tasks_outputs[task_id]:
+                        logger.info('out: ' + out.output)
 
-                if self.semaphore_helper.get_task(project_id,task_id).status == 'success':
+                if 'configure' in interface and self.semaphore_helper.get_task(project_id, task_id).status == 'success':
                     configure = interface['configure']
                     inputs = configure['inputs']
                     git_url = inputs['repository']
@@ -72,10 +76,14 @@ class AnsibleService:
                         environment_id = None
                         if env_vars:
                             environment_id = self.semaphore_helper.create_environment(project_id, name, env_vars)
-                        task_id = self.run_task(name, project_id, key_id, git_url, inventory_id, playbook_name,environment_id= environment_id)
+                        task_id = self.run_task(name, project_id, key_id, git_url, inventory_id, playbook_name,
+                                                environment_id=environment_id)
                         if self.semaphore_helper.get_task(project_id, task_id).status != 'success':
                             break
-                    tasks_outputs[task_id] = self.semaphore_helper.get_task_outputs(project_id, task_id)
+                        logger.info('playbook: ' + playbook_name + ' task_id: ' + str(task_id))
+                        tasks_outputs[task_id] = self.semaphore_helper.get_task_outputs(project_id, task_id)
+                        for out in tasks_outputs[task_id]:
+                            logger.info('out: ' + out.output)
             return tasks_outputs
 
     def build_yml_inventory(self, vms):
@@ -84,7 +92,7 @@ class AnsibleService:
         # variable_manager = VariableManager()
         inventory = {}
         all = {}
-        vars = {'ansible_ssh_common_args':'-o StrictHostKeyChecking=no'}
+        vars = {'ansible_ssh_common_args': '-o StrictHostKeyChecking=no'}
         vars['ansible_ssh_user'] = vms[0].node_template.properties['user_name']
         children = {}
         for vm in vms:
@@ -96,7 +104,7 @@ class AnsibleService:
             else:
                 hosts = children[role]
             host = {}
-            host[public_ip] =  vars
+            host[public_ip] = vars
             hosts['hosts'] = host
             children[role] = hosts
             # inventory.add_group(role)
@@ -109,22 +117,23 @@ class AnsibleService:
         private_key = vms[0].node_template.attributes['user_key_pair']['keys']['private_key']
         return base64.b64decode(private_key).decode('utf-8').replace(r'\n', '\n')
 
-    def run_task(self, name, project_id, key_id, git_url, inventory_id, playbook_name,environment_id=None):
-        logger.info('project_id: '+str(project_id)+ ' task name: ' + str(name)+ ' git url: '+git_url+' playbook: '+playbook_name)
+    def run_task(self, name, project_id, key_id, git_url, inventory_id, playbook_name, environment_id=None):
+        logger.info('project_id: ' + str(project_id) + ' task name: ' + str(
+            name) + ' git url: ' + git_url + ' playbook: ' + playbook_name)
         self.repository_id = self.semaphore_helper.create_repository(name, project_id, key_id, git_url)
         template_id = self.semaphore_helper.create_template(project_id, key_id, inventory_id, self.repository_id,
                                                             playbook_name)
-        task_id = self.semaphore_helper.execute_task(project_id, template_id, playbook_name,environment_id=environment_id)
+        task_id = self.semaphore_helper.execute_task(project_id, template_id, playbook_name,
+                                                     environment_id=environment_id)
         task = self.semaphore_helper.get_task(project_id, task_id)
-        last_output = ''
+        last_status = ''
         while task.status == 'waiting' or task.status == 'running':
             task = self.semaphore_helper.get_task(project_id, task_id)
-            logger.info('task name: '+name+ ' task status: ' + str(task.status))
-            task_outputs = self.semaphore_helper.get_task_outputs(project_id, task_id)
-            this_output = task_outputs[len(task_outputs)-1].output.replace(r'\n', '\n').replace(r'\r', '\r')
-            if last_output != this_output:
-                logger.info('task output: ' + str(this_output))
-            last_output = this_output
-            # logger.info('task output: ' + str(latask name:st_output))
+            this_status = task.status
+            if last_status != this_status:
+                logger.info('task name: ' + name + ', task status: ' + str(task.status))
+            last_status = this_status
             sleep(3)
-        return task_id
+        if 'k8s/create_k8s_dashboard.yaml'  == playbook_name:
\ No newline at end of file
+            print(playbook_name)
+        return task_id

deployer/service/deploy_service.py

 import json
+import logging
 
-from service import tosca_helper, ansible_service
+from service import tosca_helper
 from service.ansible_service import AnsibleService
-import logging
 
 logger = logging.getLogger(__name__)
 if not getattr(logger, 'handler_set', None):
@@ -24,60 +24,48 @@ class DeployService:
         self.semaphore_username = semaphore_username
         self.semaphore_password = semaphore_password
         self.vms = vms
+        for vm in vms:
+            if vm.node_template.attributes['role'] == 'master':
+                self.master_ip = vm.node_template.attributes['public_ip']
+                break
 
     def deploy(self,nodes_pair):
         target = nodes_pair[0]
         source = nodes_pair[1]
-        logger.info('target: ' + str(target) + ' source: ' + str(source))
         interface_types = tosca_helper.get_interface_types(source)
         if interface_types:
             ansible_service = AnsibleService(self.semaphore_base_url, self.semaphore_username, self.semaphore_password)
             env_vars = self.get_env_vars(nodes_pair)
             if 'Standard' in interface_types:
                 task_outputs =  ansible_service.execute(nodes_pair, 'Standard', self.vms, env_vars=env_vars)
-                self.set_attributes(task_outputs,nodes_pair)
+                nodes_pair = self.set_attributes(task_outputs,nodes_pair)
             if 'Kubernetes' in interface_types:
                 task_outputs = ansible_service.execute(nodes_pair, 'Kubernetes', self.vms, env_vars=env_vars)
-                self.set_attributes(task_outputs,nodes_pair)
+                nodes_pair = self.set_attributes(task_outputs,nodes_pair)
 
-        return None
+        return nodes_pair
 
     def get_env_vars(self, nodes_pair):
         target = nodes_pair[0]
         source = nodes_pair[1]
+        env_vars = {'K8s_NAMESPACE': 'default'}
         if source.node_template.type == 'tosca.nodes.QC.Container.Application.Docker':
             env_vars = {'DOCKER_IMAGE':source.node_template.artifacts['image']['file']}
             env_vars['DOCKER_SERVICE_NAME'] = source.name
             env_vars['CONTAINER_PORT'] = source.node_template.properties['ports'][0].split(':')[1]
-            env_vars['K8s_NAMESPACE'] = 'default'
-            return env_vars
-        return None
+        return env_vars
 
     def set_attributes(self, task_outputs,nodes_pair):
         target = nodes_pair[0]
         source = nodes_pair[1]
         if source.node_template.type == 'tosca.nodes.QC.docker.Orchestrator.Kubernetes':
-            if 'tokens' not in source.node_template.attributes:
-                tokens = []
-                source.node_template.attributes['tokens'] = tokens
-            else:
-                tokens = source.node_template.attributes['tokens']
-
-            k8s_secrets = None
-            k8s_services = None
-            for task_output_key in task_outputs:
-                task_output = task_outputs[task_output_key]
-                if not k8s_secrets:
-                    k8s_secrets = self.parse_ansible_var('k8s_secrets',task_output)
-                if not k8s_services:
-                    k8s_services = self.parse_ansible_var('k8s_services', task_output)
-                if k8s_services and k8s_secrets:
-                    break
+            source = self.set_kubernetes_attributes(source=source,task_outputs=task_outputs)
+            lst = list(nodes_pair)
+            lst[1] = source
+            nodes_pair = tuple(lst)
+        return nodes_pair
 
 
-            credential = {'token_type':'k8s_dashboard_token'}
-            credential['token'] = self.get_dashboard_token(k8s_secrets)
-            tokens.append(credential)
 
     def parse_ansible_var(self,var_name, output_array):
         index = 0
@@ -85,20 +73,82 @@ class DeployService:
         end_index = -1
         for out in output_array:
             index += 1
-            if 'TASK' in out or 'PLAY RECAP' in out:
+            if 'TASK' in out.output or 'PLAY RECAP' in out.output:
                 if start_index > -1:
                     end_index = index - 1
                     break
-            if '"' + var_name + '":' in out:
+            if start_index <=-1 and '"' + var_name + '":' in out.output:
                 start_index = index - 1
         if start_index <= -1:
             return None
         ansible_var = output_array[start_index:end_index]
         json_ansible_var = '{'
         for item in ansible_var:
-            json_ansible_var = json_ansible_var + item
+            json_ansible_var = json_ansible_var + item.output
+        logger.info('found '+var_name +': '+ str(json_ansible_var))
         return json.loads(json_ansible_var)
 
-    def get_dashboard_token(self, k8s_secrets):
-        print(k8s_secrets)
-        return None
+    def get_dashboard_token(self, k8s_dashboard_token):
+        k8s_dashboard_token = k8s_dashboard_token['k8s_dashboard_token']
+        # if 'resources' in k8s_secrets:
\ No newline at end of file
+        #     return self.get_secret_from_k8s_info(k8s_secrets)
+        if 'stdout' in k8s_dashboard_token:
+            return self.get_secret_from_stdout(k8s_dashboard_token)
+
+
+    def get_service_port(self, k8s_services, service_name,port_type):
+        resources = k8s_services['k8s_services']['resources']
+        for resource in resources:
+            name = resource['metadata']['name']
+            if name == service_name:
+                ports = resource['spec']['ports']
+                for port in ports:
+                    if port_type in port:
+                        return port[port_type]
+        return None
+
+    def get_secret_from_k8s_info(self, k8s_secrets):
+        resources = k8s_secrets['resources']
+        for resource in resources:
+            metadata = resource['metadata']
+            if 'admin-user-token' in  metadata['name']:
+                dashboard_token = resource['data']['token']
+                logger.info('found dashboard_token: ' + str(dashboard_token))
+                return resource['data']['token']
+        return None
+
+    def get_secret_from_stdout(self, k8s_dashboard_token):
+        return k8s_dashboard_token['stdout'].replace('token:     ', '')
+
+    def set_kubernetes_attributes(self, source=None,task_outputs=None):
+        attributes = source.node_template.attributes
+        if 'tokens' not in attributes:
+            tokens = []
+            attributes['tokens'] = tokens
+        else:
+            tokens = attributes['tokens']
+
+        if 'dashboard_url' not in source.node_template.attributes:
+            dashboard_url = ''
+            attributes['dashboard_url'] = tokens
+        else:
+            dashboard_url = attributes['dashboard_url']
+
+        k8s_dashboard_token = None
+        k8s_services = None
+        for task_output_key in task_outputs:
+            task_output = task_outputs[task_output_key]
+            if not k8s_dashboard_token:
+                k8s_dashboard_token = self.parse_ansible_var('k8s_dashboard_token', task_output)
+            if not k8s_services:
+                k8s_services = self.parse_ansible_var('k8s_services', task_output)
+            if k8s_services and k8s_dashboard_token:
+                credential = {'token_type': 'k8s_dashboard_token'}
+                credential['token'] = self.get_dashboard_token(k8s_dashboard_token)
+                tokens.append(credential)
+
+                service_port = self.get_service_port(k8s_services, 'kubernetes-dashboard', 'nodePort')
+                dashboard_url = 'https://' + self.master_ip + ':' + str(service_port)
+                attributes['dashboard_url'] = dashboard_url
+                logger.info('source.node_template.attributes: ' + str(attributes))
+                return source

deployer/service/tosca_helper.py

@@ -28,7 +28,6 @@ class ToscaHelper:
         return api
 
     def get_interface_types(target):
-        print(target.node_template.interfaces)
         interface_types = []
         for interface in target.node_template.interfaces:
             interface_types.append(interface)
@@ -73,6 +72,13 @@ class ToscaHelper:
     def get_vms(self):
         return self.tosca_client.get_node_templates(self.doc_id, type_name='tosca.nodes.QC.VM.Compute')
 
+    def set_node(self, updated_node, tosca_template_dict):
+        node_templates = tosca_template_dict['topology_template']['node_templates']
+        for node_name in node_templates:
+            if node_name == updated_node.name:
+                node_templates[node_name] = updated_node.node_template
+                return tosca_template_dict
+
 
 def get_interface_types(node):
     interface_type_names = []

deployer/test/test_deployer.py

@@ -52,8 +52,9 @@ class TestDeployer(unittest.TestCase):
                 self.assertIsNotNone(nodes_pairs)
 
                 username = 'admin'
-                deployService = DeployService(polemarch_base_url=polemarch_base_url,polemarch_username=username,polemarch_password='admin',
-                                              semaphore_base_url=semaphore_base_url,semaphore_username=username,semaphore_password='password')
+                deployService = DeployService(polemarch_username=username,polemarch_password='admin',
+                                              semaphore_base_url=semaphore_base_url,semaphore_username=username,
+                                              semaphore_password='password')
                 for node_pair in nodes_pairs:
                     deployService.deploy(node_pair)