Commit 8e9dc5fb authored by Spiros Koulouzis's avatar Spiros Koulouzis

fix error with indentation: we where not storing the outputs of some tasks

parent 728e675c
...@@ -8,14 +8,16 @@ import os ...@@ -8,14 +8,16 @@ import os
import sys import sys
import tempfile import tempfile
import time import time
import traceback
from threading import Thread from threading import Thread
from time import sleep from time import sleep
from service.deploy_service import DeployService
from service.tosca_helper import ToscaHelper
import pika import pika
import sure_tosca_client
import yaml import yaml
from service.deploy_service import DeployService
from service.tosca_helper import ToscaHelper
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
done = False done = False
...@@ -85,9 +87,21 @@ def handle_delivery(message): ...@@ -85,9 +87,21 @@ def handle_delivery(message):
nodes_pairs = tosca_helper.get_deployment_node_pairs() nodes_pairs = tosca_helper.get_deployment_node_pairs()
deployService = DeployService(semaphore_base_url=semaphore_base_url, semaphore_username=semaphore_username, deployService = DeployService(semaphore_base_url=semaphore_base_url, semaphore_username=semaphore_username,
semaphore_password=semaphore_password,vms=tosca_helper.get_vms()) semaphore_password=semaphore_password, vms=tosca_helper.get_vms())
for node_pair in nodes_pairs: try:
deployService.deploy(node_pair) for node_pair in nodes_pairs:
updated_node_pairs = deployService.deploy(node_pair)
for updated_node in updated_node_pairs:
if isinstance(updated_node, list):
for node in updated_node:
tosca_helper.set_node(node,tosca_template_dict)
else:
tosca_helper.set_node(updated_node, tosca_template_dict)
except Exception as e:
track = traceback.format_exc()
print(track)
raise
response = {'toscaTemplate': tosca_template_dict} response = {'toscaTemplate': tosca_template_dict}
output_current_milli_time = int(round(time.time() * 1000)) output_current_milli_time = int(round(time.time() * 1000))
......
...@@ -6,7 +6,7 @@ import datetime ...@@ -6,7 +6,7 @@ import datetime
import yaml import yaml
from semaphore_client.semaphore_helper import SemaphoreHelper from semaphore_client.semaphore_helper import SemaphoreHelper
yaml.Dumper.ignore_aliases = lambda *args : True yaml.Dumper.ignore_aliases = lambda *args: True
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
if not getattr(logger, 'handler_set', None): if not getattr(logger, 'handler_set', None):
...@@ -20,16 +20,15 @@ if not getattr(logger, 'handler_set', None): ...@@ -20,16 +20,15 @@ if not getattr(logger, 'handler_set', None):
class AnsibleService: class AnsibleService:
def __init__(self, semaphore_base_url=None,semaphore_username=None,semaphore_password=None): def __init__(self, semaphore_base_url=None, semaphore_username=None, semaphore_password=None):
self.semaphore_base_url = semaphore_base_url self.semaphore_base_url = semaphore_base_url
self.semaphore_username = semaphore_username self.semaphore_username = semaphore_username
self.semaphore_password = semaphore_password self.semaphore_password = semaphore_password
self.semaphore_helper = SemaphoreHelper(self.semaphore_base_url, self.semaphore_username, self.semaphore_password) self.semaphore_helper = SemaphoreHelper(self.semaphore_base_url, self.semaphore_username,
self.semaphore_password)
self.repository_id = None self.repository_id = None
self.template_id = None self.template_id = None
def execute(self, nodes_pair, interface_type, vms, env_vars=None): def execute(self, nodes_pair, interface_type, vms, env_vars=None):
application = nodes_pair[1] application = nodes_pair[1]
name = application.name name = application.name
...@@ -43,11 +42,12 @@ class AnsibleService: ...@@ -43,11 +42,12 @@ class AnsibleService:
if desired_state: if desired_state:
now = datetime.datetime.now() now = datetime.datetime.now()
project_id = self.semaphore_helper.create_project(application.name+'_'+str(now)) project_id = self.semaphore_helper.create_project(application.name + '_' + str(now))
inventory_contents = yaml.dump( self.build_yml_inventory(vms),default_flow_style=False) inventory_contents = yaml.dump(self.build_yml_inventory(vms), default_flow_style=False)
private_key = self.get_private_key(vms) private_key = self.get_private_key(vms)
key_id = self.semaphore_helper.create_ssh_key(application.name, project_id, private_key) key_id = self.semaphore_helper.create_ssh_key(application.name, project_id, private_key)
inventory_id = self.semaphore_helper.create_inventory(application.name, project_id, key_id,inventory_contents) inventory_id = self.semaphore_helper.create_inventory(application.name, project_id, key_id,
inventory_contents)
if 'RUNNING' == desired_state: if 'RUNNING' == desired_state:
interface = interfaces[interface_type] interface = interfaces[interface_type]
create = interface['create'] create = interface['create']
...@@ -57,13 +57,17 @@ class AnsibleService: ...@@ -57,13 +57,17 @@ class AnsibleService:
for playbook_name in playbook_names: for playbook_name in playbook_names:
environment_id = None environment_id = None
if env_vars: if env_vars:
environment_id = self.semaphore_helper.create_environment(project_id,name,env_vars) environment_id = self.semaphore_helper.create_environment(project_id, name, env_vars)
task_id = self.run_task(name, project_id, key_id, git_url, inventory_id, playbook_name, environment_id= environment_id) task_id = self.run_task(name, project_id, key_id, git_url, inventory_id, playbook_name,
environment_id=environment_id)
if self.semaphore_helper.get_task(project_id, task_id).status != 'success': if self.semaphore_helper.get_task(project_id, task_id).status != 'success':
break break
logger.info('playbook: ' + playbook_name + ' task_id: ' + str(task_id))
tasks_outputs[task_id] = self.semaphore_helper.get_task_outputs(project_id, task_id) tasks_outputs[task_id] = self.semaphore_helper.get_task_outputs(project_id, task_id)
for out in tasks_outputs[task_id]:
logger.info('out: ' + out.output)
if self.semaphore_helper.get_task(project_id,task_id).status == 'success': if 'configure' in interface and self.semaphore_helper.get_task(project_id, task_id).status == 'success':
configure = interface['configure'] configure = interface['configure']
inputs = configure['inputs'] inputs = configure['inputs']
git_url = inputs['repository'] git_url = inputs['repository']
...@@ -72,10 +76,14 @@ class AnsibleService: ...@@ -72,10 +76,14 @@ class AnsibleService:
environment_id = None environment_id = None
if env_vars: if env_vars:
environment_id = self.semaphore_helper.create_environment(project_id, name, env_vars) environment_id = self.semaphore_helper.create_environment(project_id, name, env_vars)
task_id = self.run_task(name, project_id, key_id, git_url, inventory_id, playbook_name,environment_id= environment_id) task_id = self.run_task(name, project_id, key_id, git_url, inventory_id, playbook_name,
environment_id=environment_id)
if self.semaphore_helper.get_task(project_id, task_id).status != 'success': if self.semaphore_helper.get_task(project_id, task_id).status != 'success':
break break
tasks_outputs[task_id] = self.semaphore_helper.get_task_outputs(project_id, task_id) logger.info('playbook: ' + playbook_name + ' task_id: ' + str(task_id))
tasks_outputs[task_id] = self.semaphore_helper.get_task_outputs(project_id, task_id)
for out in tasks_outputs[task_id]:
logger.info('out: ' + out.output)
return tasks_outputs return tasks_outputs
def build_yml_inventory(self, vms): def build_yml_inventory(self, vms):
...@@ -84,7 +92,7 @@ class AnsibleService: ...@@ -84,7 +92,7 @@ class AnsibleService:
# variable_manager = VariableManager() # variable_manager = VariableManager()
inventory = {} inventory = {}
all = {} all = {}
vars = {'ansible_ssh_common_args':'-o StrictHostKeyChecking=no'} vars = {'ansible_ssh_common_args': '-o StrictHostKeyChecking=no'}
vars['ansible_ssh_user'] = vms[0].node_template.properties['user_name'] vars['ansible_ssh_user'] = vms[0].node_template.properties['user_name']
children = {} children = {}
for vm in vms: for vm in vms:
...@@ -96,7 +104,7 @@ class AnsibleService: ...@@ -96,7 +104,7 @@ class AnsibleService:
else: else:
hosts = children[role] hosts = children[role]
host = {} host = {}
host[public_ip] = vars host[public_ip] = vars
hosts['hosts'] = host hosts['hosts'] = host
children[role] = hosts children[role] = hosts
# inventory.add_group(role) # inventory.add_group(role)
...@@ -109,22 +117,23 @@ class AnsibleService: ...@@ -109,22 +117,23 @@ class AnsibleService:
private_key = vms[0].node_template.attributes['user_key_pair']['keys']['private_key'] private_key = vms[0].node_template.attributes['user_key_pair']['keys']['private_key']
return base64.b64decode(private_key).decode('utf-8').replace(r'\n', '\n') return base64.b64decode(private_key).decode('utf-8').replace(r'\n', '\n')
def run_task(self, name, project_id, key_id, git_url, inventory_id, playbook_name,environment_id=None): def run_task(self, name, project_id, key_id, git_url, inventory_id, playbook_name, environment_id=None):
logger.info('project_id: '+str(project_id)+ ' task name: ' + str(name)+ ' git url: '+git_url+' playbook: '+playbook_name) logger.info('project_id: ' + str(project_id) + ' task name: ' + str(
name) + ' git url: ' + git_url + ' playbook: ' + playbook_name)
self.repository_id = self.semaphore_helper.create_repository(name, project_id, key_id, git_url) self.repository_id = self.semaphore_helper.create_repository(name, project_id, key_id, git_url)
template_id = self.semaphore_helper.create_template(project_id, key_id, inventory_id, self.repository_id, template_id = self.semaphore_helper.create_template(project_id, key_id, inventory_id, self.repository_id,
playbook_name) playbook_name)
task_id = self.semaphore_helper.execute_task(project_id, template_id, playbook_name,environment_id=environment_id) task_id = self.semaphore_helper.execute_task(project_id, template_id, playbook_name,
environment_id=environment_id)
task = self.semaphore_helper.get_task(project_id, task_id) task = self.semaphore_helper.get_task(project_id, task_id)
last_output = '' last_status = ''
while task.status == 'waiting' or task.status == 'running': while task.status == 'waiting' or task.status == 'running':
task = self.semaphore_helper.get_task(project_id, task_id) task = self.semaphore_helper.get_task(project_id, task_id)
logger.info('task name: '+name+ ' task status: ' + str(task.status)) this_status = task.status
task_outputs = self.semaphore_helper.get_task_outputs(project_id, task_id) if last_status != this_status:
this_output = task_outputs[len(task_outputs)-1].output.replace(r'\n', '\n').replace(r'\r', '\r') logger.info('task name: ' + name + ', task status: ' + str(task.status))
if last_output != this_output: last_status = this_status
logger.info('task output: ' + str(this_output))
last_output = this_output
# logger.info('task output: ' + str(latask name:st_output))
sleep(3) sleep(3)
return task_id if 'k8s/create_k8s_dashboard.yaml' == playbook_name:
\ No newline at end of file print(playbook_name)
return task_id
import json import json
import logging
from service import tosca_helper, ansible_service from service import tosca_helper
from service.ansible_service import AnsibleService from service.ansible_service import AnsibleService
import logging
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
if not getattr(logger, 'handler_set', None): if not getattr(logger, 'handler_set', None):
...@@ -24,60 +24,48 @@ class DeployService: ...@@ -24,60 +24,48 @@ class DeployService:
self.semaphore_username = semaphore_username self.semaphore_username = semaphore_username
self.semaphore_password = semaphore_password self.semaphore_password = semaphore_password
self.vms = vms self.vms = vms
for vm in vms:
if vm.node_template.attributes['role'] == 'master':
self.master_ip = vm.node_template.attributes['public_ip']
break
def deploy(self,nodes_pair): def deploy(self,nodes_pair):
target = nodes_pair[0] target = nodes_pair[0]
source = nodes_pair[1] source = nodes_pair[1]
logger.info('target: ' + str(target) + ' source: ' + str(source))
interface_types = tosca_helper.get_interface_types(source) interface_types = tosca_helper.get_interface_types(source)
if interface_types: if interface_types:
ansible_service = AnsibleService(self.semaphore_base_url, self.semaphore_username, self.semaphore_password) ansible_service = AnsibleService(self.semaphore_base_url, self.semaphore_username, self.semaphore_password)
env_vars = self.get_env_vars(nodes_pair) env_vars = self.get_env_vars(nodes_pair)
if 'Standard' in interface_types: if 'Standard' in interface_types:
task_outputs = ansible_service.execute(nodes_pair, 'Standard', self.vms, env_vars=env_vars) task_outputs = ansible_service.execute(nodes_pair, 'Standard', self.vms, env_vars=env_vars)
self.set_attributes(task_outputs,nodes_pair) nodes_pair = self.set_attributes(task_outputs,nodes_pair)
if 'Kubernetes' in interface_types: if 'Kubernetes' in interface_types:
task_outputs = ansible_service.execute(nodes_pair, 'Kubernetes', self.vms, env_vars=env_vars) task_outputs = ansible_service.execute(nodes_pair, 'Kubernetes', self.vms, env_vars=env_vars)
self.set_attributes(task_outputs,nodes_pair) nodes_pair = self.set_attributes(task_outputs,nodes_pair)
return None return nodes_pair
def get_env_vars(self, nodes_pair): def get_env_vars(self, nodes_pair):
target = nodes_pair[0] target = nodes_pair[0]
source = nodes_pair[1] source = nodes_pair[1]
env_vars = {'K8s_NAMESPACE': 'default'}
if source.node_template.type == 'tosca.nodes.QC.Container.Application.Docker': if source.node_template.type == 'tosca.nodes.QC.Container.Application.Docker':
env_vars = {'DOCKER_IMAGE':source.node_template.artifacts['image']['file']} env_vars = {'DOCKER_IMAGE':source.node_template.artifacts['image']['file']}
env_vars['DOCKER_SERVICE_NAME'] = source.name env_vars['DOCKER_SERVICE_NAME'] = source.name
env_vars['CONTAINER_PORT'] = source.node_template.properties['ports'][0].split(':')[1] env_vars['CONTAINER_PORT'] = source.node_template.properties['ports'][0].split(':')[1]
env_vars['K8s_NAMESPACE'] = 'default' return env_vars
return env_vars
return None
def set_attributes(self, task_outputs,nodes_pair): def set_attributes(self, task_outputs,nodes_pair):
target = nodes_pair[0] target = nodes_pair[0]
source = nodes_pair[1] source = nodes_pair[1]
if source.node_template.type == 'tosca.nodes.QC.docker.Orchestrator.Kubernetes': if source.node_template.type == 'tosca.nodes.QC.docker.Orchestrator.Kubernetes':
if 'tokens' not in source.node_template.attributes: source = self.set_kubernetes_attributes(source=source,task_outputs=task_outputs)
tokens = [] lst = list(nodes_pair)
source.node_template.attributes['tokens'] = tokens lst[1] = source
else: nodes_pair = tuple(lst)
tokens = source.node_template.attributes['tokens'] return nodes_pair
k8s_secrets = None
k8s_services = None
for task_output_key in task_outputs:
task_output = task_outputs[task_output_key]
if not k8s_secrets:
k8s_secrets = self.parse_ansible_var('k8s_secrets',task_output)
if not k8s_services:
k8s_services = self.parse_ansible_var('k8s_services', task_output)
if k8s_services and k8s_secrets:
break
credential = {'token_type':'k8s_dashboard_token'}
credential['token'] = self.get_dashboard_token(k8s_secrets)
tokens.append(credential)
def parse_ansible_var(self,var_name, output_array): def parse_ansible_var(self,var_name, output_array):
index = 0 index = 0
...@@ -85,20 +73,82 @@ class DeployService: ...@@ -85,20 +73,82 @@ class DeployService:
end_index = -1 end_index = -1
for out in output_array: for out in output_array:
index += 1 index += 1
if 'TASK' in out or 'PLAY RECAP' in out: if 'TASK' in out.output or 'PLAY RECAP' in out.output:
if start_index > -1: if start_index > -1:
end_index = index - 1 end_index = index - 1
break break
if '"' + var_name + '":' in out: if start_index <=-1 and '"' + var_name + '":' in out.output:
start_index = index - 1 start_index = index - 1
if start_index <= -1: if start_index <= -1:
return None return None
ansible_var = output_array[start_index:end_index] ansible_var = output_array[start_index:end_index]
json_ansible_var = '{' json_ansible_var = '{'
for item in ansible_var: for item in ansible_var:
json_ansible_var = json_ansible_var + item json_ansible_var = json_ansible_var + item.output
logger.info('found '+var_name +': '+ str(json_ansible_var))
return json.loads(json_ansible_var) return json.loads(json_ansible_var)
def get_dashboard_token(self, k8s_secrets): def get_dashboard_token(self, k8s_dashboard_token):
print(k8s_secrets) k8s_dashboard_token = k8s_dashboard_token['k8s_dashboard_token']
return None # if 'resources' in k8s_secrets:
\ No newline at end of file # return self.get_secret_from_k8s_info(k8s_secrets)
if 'stdout' in k8s_dashboard_token:
return self.get_secret_from_stdout(k8s_dashboard_token)
def get_service_port(self, k8s_services, service_name,port_type):
resources = k8s_services['k8s_services']['resources']
for resource in resources:
name = resource['metadata']['name']
if name == service_name:
ports = resource['spec']['ports']
for port in ports:
if port_type in port:
return port[port_type]
return None
def get_secret_from_k8s_info(self, k8s_secrets):
resources = k8s_secrets['resources']
for resource in resources:
metadata = resource['metadata']
if 'admin-user-token' in metadata['name']:
dashboard_token = resource['data']['token']
logger.info('found dashboard_token: ' + str(dashboard_token))
return resource['data']['token']
return None
def get_secret_from_stdout(self, k8s_dashboard_token):
return k8s_dashboard_token['stdout'].replace('token: ', '')
def set_kubernetes_attributes(self, source=None,task_outputs=None):
attributes = source.node_template.attributes
if 'tokens' not in attributes:
tokens = []
attributes['tokens'] = tokens
else:
tokens = attributes['tokens']
if 'dashboard_url' not in source.node_template.attributes:
dashboard_url = ''
attributes['dashboard_url'] = tokens
else:
dashboard_url = attributes['dashboard_url']
k8s_dashboard_token = None
k8s_services = None
for task_output_key in task_outputs:
task_output = task_outputs[task_output_key]
if not k8s_dashboard_token:
k8s_dashboard_token = self.parse_ansible_var('k8s_dashboard_token', task_output)
if not k8s_services:
k8s_services = self.parse_ansible_var('k8s_services', task_output)
if k8s_services and k8s_dashboard_token:
credential = {'token_type': 'k8s_dashboard_token'}
credential['token'] = self.get_dashboard_token(k8s_dashboard_token)
tokens.append(credential)
service_port = self.get_service_port(k8s_services, 'kubernetes-dashboard', 'nodePort')
dashboard_url = 'https://' + self.master_ip + ':' + str(service_port)
attributes['dashboard_url'] = dashboard_url
logger.info('source.node_template.attributes: ' + str(attributes))
return source
...@@ -28,7 +28,6 @@ class ToscaHelper: ...@@ -28,7 +28,6 @@ class ToscaHelper:
return api return api
def get_interface_types(target): def get_interface_types(target):
print(target.node_template.interfaces)
interface_types = [] interface_types = []
for interface in target.node_template.interfaces: for interface in target.node_template.interfaces:
interface_types.append(interface) interface_types.append(interface)
...@@ -73,6 +72,13 @@ class ToscaHelper: ...@@ -73,6 +72,13 @@ class ToscaHelper:
def get_vms(self): def get_vms(self):
return self.tosca_client.get_node_templates(self.doc_id, type_name='tosca.nodes.QC.VM.Compute') return self.tosca_client.get_node_templates(self.doc_id, type_name='tosca.nodes.QC.VM.Compute')
def set_node(self, updated_node, tosca_template_dict):
node_templates = tosca_template_dict['topology_template']['node_templates']
for node_name in node_templates:
if node_name == updated_node.name:
node_templates[node_name] = updated_node.node_template
return tosca_template_dict
def get_interface_types(node): def get_interface_types(node):
interface_type_names = [] interface_type_names = []
......
...@@ -52,8 +52,9 @@ class TestDeployer(unittest.TestCase): ...@@ -52,8 +52,9 @@ class TestDeployer(unittest.TestCase):
self.assertIsNotNone(nodes_pairs) self.assertIsNotNone(nodes_pairs)
username = 'admin' username = 'admin'
deployService = DeployService(polemarch_base_url=polemarch_base_url,polemarch_username=username,polemarch_password='admin', deployService = DeployService(polemarch_username=username,polemarch_password='admin',
semaphore_base_url=semaphore_base_url,semaphore_username=username,semaphore_password='password') semaphore_base_url=semaphore_base_url,semaphore_username=username,
semaphore_password='password')
for node_pair in nodes_pairs: for node_pair in nodes_pairs:
deployService.deploy(node_pair) deployService.deploy(node_pair)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment