Skip to content

C
CONF
Commit 8e9dc5fb authored by Spiros Koulouzis's avatar Spiros Koulouzis
Browse files
Options

fix error with indentation: we where not storing the outputs of some tasks

parent 728e675c
Hide whitespace changes
Inline Side-by-side
Showing with 150 additions and 70 deletions
deployer/__main__.py
View file @ 8e9dc5fb
......@@ -8,14 +8,16 @@ import os
import sys
import tempfile
import time
import traceback
from threading import Thread
from time import sleep
from service.deploy_service import DeployService
from service.tosca_helper import ToscaHelper
import pika
import sure_tosca_client
import yaml
from service.deploy_service import DeployService
from service.tosca_helper import ToscaHelper
logger = logging.getLogger(__name__)
done = False
......@@ -85,9 +87,21 @@ def handle_delivery(message):
nodes_pairs = tosca_helper.get_deployment_node_pairs()
deployService = DeployService(semaphore_base_url=semaphore_base_url, semaphore_username=semaphore_username,
semaphore_password=semaphore_password,vms=tosca_helper.get_vms())
for node_pair in nodes_pairs:
deployService.deploy(node_pair)
semaphore_password=semaphore_password, vms=tosca_helper.get_vms())
try:
for node_pair in nodes_pairs:
updated_node_pairs = deployService.deploy(node_pair)
for updated_node in updated_node_pairs:
if isinstance(updated_node, list):
for node in updated_node:
tosca_helper.set_node(node,tosca_template_dict)
else:
tosca_helper.set_node(updated_node, tosca_template_dict)
except Exception as e:
track = traceback.format_exc()
print(track)
raise
response = {'toscaTemplate': tosca_template_dict}
output_current_milli_time = int(round(time.time() * 1000))
......
deployer/service/ansible_service.py
View file @ 8e9dc5fb
......@@ -6,7 +6,7 @@ import datetime
import yaml
from semaphore_client.semaphore_helper import SemaphoreHelper
yaml.Dumper.ignore_aliases = lambda *args : True
yaml.Dumper.ignore_aliases = lambda *args: True
logger = logging.getLogger(__name__)
if not getattr(logger, 'handler_set', None):
......@@ -20,16 +20,15 @@ if not getattr(logger, 'handler_set', None):
class AnsibleService:
def __init__(self, semaphore_base_url=None,semaphore_username=None,semaphore_password=None):
def __init__(self, semaphore_base_url=None, semaphore_username=None, semaphore_password=None):
self.semaphore_base_url = semaphore_base_url
self.semaphore_username = semaphore_username
self.semaphore_password = semaphore_password
self.semaphore_helper = SemaphoreHelper(self.semaphore_base_url, self.semaphore_username, self.semaphore_password)
self.semaphore_helper = SemaphoreHelper(self.semaphore_base_url, self.semaphore_username,
self.semaphore_password)
self.repository_id = None
self.template_id = None
def execute(self, nodes_pair, interface_type, vms, env_vars=None):
application = nodes_pair[1]
name = application.name
......@@ -43,11 +42,12 @@ class AnsibleService:
if desired_state:
now = datetime.datetime.now()
project_id = self.semaphore_helper.create_project(application.name+'_'+str(now))
inventory_contents = yaml.dump( self.build_yml_inventory(vms),default_flow_style=False)
project_id = self.semaphore_helper.create_project(application.name + '_' + str(now))
inventory_contents = yaml.dump(self.build_yml_inventory(vms), default_flow_style=False)
private_key = self.get_private_key(vms)
key_id = self.semaphore_helper.create_ssh_key(application.name, project_id, private_key)
inventory_id = self.semaphore_helper.create_inventory(application.name, project_id, key_id,inventory_contents)
inventory_id = self.semaphore_helper.create_inventory(application.name, project_id, key_id,
inventory_contents)
if 'RUNNING' == desired_state:
interface = interfaces[interface_type]
create = interface['create']
......@@ -57,13 +57,17 @@ class AnsibleService:
for playbook_name in playbook_names:
environment_id = None
if env_vars:
environment_id = self.semaphore_helper.create_environment(project_id,name,env_vars)
task_id = self.run_task(name, project_id, key_id, git_url, inventory_id, playbook_name, environment_id= environment_id)
environment_id = self.semaphore_helper.create_environment(project_id, name, env_vars)
task_id = self.run_task(name, project_id, key_id, git_url, inventory_id, playbook_name,
environment_id=environment_id)
if self.semaphore_helper.get_task(project_id, task_id).status != 'success':
break
logger.info('playbook: ' + playbook_name + ' task_id: ' + str(task_id))
tasks_outputs[task_id] = self.semaphore_helper.get_task_outputs(project_id, task_id)
for out in tasks_outputs[task_id]:
logger.info('out: ' + out.output)
if self.semaphore_helper.get_task(project_id,task_id).status == 'success':
if 'configure' in interface and self.semaphore_helper.get_task(project_id, task_id).status == 'success':
configure = interface['configure']
inputs = configure['inputs']
git_url = inputs['repository']
......@@ -72,10 +76,14 @@ class AnsibleService:
environment_id = None
if env_vars:
environment_id = self.semaphore_helper.create_environment(project_id, name, env_vars)
task_id = self.run_task(name, project_id, key_id, git_url, inventory_id, playbook_name,environment_id= environment_id)
task_id = self.run_task(name, project_id, key_id, git_url, inventory_id, playbook_name,
environment_id=environment_id)
if self.semaphore_helper.get_task(project_id, task_id).status != 'success':
break
tasks_outputs[task_id] = self.semaphore_helper.get_task_outputs(project_id, task_id)
logger.info('playbook: ' + playbook_name + ' task_id: ' + str(task_id))
tasks_outputs[task_id] = self.semaphore_helper.get_task_outputs(project_id, task_id)
for out in tasks_outputs[task_id]:
logger.info('out: ' + out.output)
return tasks_outputs
def build_yml_inventory(self, vms):
......@@ -84,7 +92,7 @@ class AnsibleService:
# variable_manager = VariableManager()
inventory = {}
all = {}
vars = {'ansible_ssh_common_args':'-o StrictHostKeyChecking=no'}
vars = {'ansible_ssh_common_args': '-o StrictHostKeyChecking=no'}
vars['ansible_ssh_user'] = vms[0].node_template.properties['user_name']
children = {}
for vm in vms:
......@@ -96,7 +104,7 @@ class AnsibleService:
else:
hosts = children[role]
host = {}
host[public_ip] = vars
host[public_ip] = vars
hosts['hosts'] = host
children[role] = hosts
# inventory.add_group(role)
......@@ -109,22 +117,23 @@ class AnsibleService:
private_key = vms[0].node_template.attributes['user_key_pair']['keys']['private_key']
return base64.b64decode(private_key).decode('utf-8').replace(r'\n', '\n')
def run_task(self, name, project_id, key_id, git_url, inventory_id, playbook_name,environment_id=None):
logger.info('project_id: '+str(project_id)+ ' task name: ' + str(name)+ ' git url: '+git_url+' playbook: '+playbook_name)
def run_task(self, name, project_id, key_id, git_url, inventory_id, playbook_name, environment_id=None):
logger.info('project_id: ' + str(project_id) + ' task name: ' + str(
name) + ' git url: ' + git_url + ' playbook: ' + playbook_name)
self.repository_id = self.semaphore_helper.create_repository(name, project_id, key_id, git_url)
template_id = self.semaphore_helper.create_template(project_id, key_id, inventory_id, self.repository_id,
playbook_name)
task_id = self.semaphore_helper.execute_task(project_id, template_id, playbook_name,environment_id=environment_id)
task_id = self.semaphore_helper.execute_task(project_id, template_id, playbook_name,
environment_id=environment_id)
task = self.semaphore_helper.get_task(project_id, task_id)
last_output = ''
last_status = ''
while task.status == 'waiting' or task.status == 'running':
task = self.semaphore_helper.get_task(project_id, task_id)
logger.info('task name: '+name+ ' task status: ' + str(task.status))
task_outputs = self.semaphore_helper.get_task_outputs(project_id, task_id)
this_output = task_outputs[len(task_outputs)-1].output.replace(r'\n', '\n').replace(r'\r', '\r')
if last_output != this_output:
logger.info('task output: ' + str(this_output))
last_output = this_output
# logger.info('task output: ' + str(latask name:st_output))
this_status = task.status
if last_status != this_status:
logger.info('task name: ' + name + ', task status: ' + str(task.status))
last_status = this_status
sleep(3)
return task_id
\ No newline at end of file
if 'k8s/create_k8s_dashboard.yaml' == playbook_name:
print(playbook_name)
return task_id
deployer/service/deploy_service.py
View file @ 8e9dc5fb
import json
import logging
from service import tosca_helper, ansible_service
from service import tosca_helper
from service.ansible_service import AnsibleService
import logging
logger = logging.getLogger(__name__)
if not getattr(logger, 'handler_set', None):
......@@ -24,60 +24,48 @@ class DeployService:
self.semaphore_username = semaphore_username
self.semaphore_password = semaphore_password
self.vms = vms
for vm in vms:
if vm.node_template.attributes['role'] == 'master':
self.master_ip = vm.node_template.attributes['public_ip']
break
def deploy(self,nodes_pair):
target = nodes_pair[0]
source = nodes_pair[1]
logger.info('target: ' + str(target) + ' source: ' + str(source))
interface_types = tosca_helper.get_interface_types(source)
if interface_types:
ansible_service = AnsibleService(self.semaphore_base_url, self.semaphore_username, self.semaphore_password)
env_vars = self.get_env_vars(nodes_pair)
if 'Standard' in interface_types:
task_outputs = ansible_service.execute(nodes_pair, 'Standard', self.vms, env_vars=env_vars)
self.set_attributes(task_outputs,nodes_pair)
nodes_pair = self.set_attributes(task_outputs,nodes_pair)
if 'Kubernetes' in interface_types:
task_outputs = ansible_service.execute(nodes_pair, 'Kubernetes', self.vms, env_vars=env_vars)
self.set_attributes(task_outputs,nodes_pair)
nodes_pair = self.set_attributes(task_outputs,nodes_pair)
return None
return nodes_pair
def get_env_vars(self, nodes_pair):
target = nodes_pair[0]
source = nodes_pair[1]
env_vars = {'K8s_NAMESPACE': 'default'}
if source.node_template.type == 'tosca.nodes.QC.Container.Application.Docker':
env_vars = {'DOCKER_IMAGE':source.node_template.artifacts['image']['file']}
env_vars['DOCKER_SERVICE_NAME'] = source.name
env_vars['CONTAINER_PORT'] = source.node_template.properties['ports'][0].split(':')[1]
env_vars['K8s_NAMESPACE'] = 'default'
return env_vars
return None
return env_vars
def set_attributes(self, task_outputs,nodes_pair):
target = nodes_pair[0]
source = nodes_pair[1]
if source.node_template.type == 'tosca.nodes.QC.docker.Orchestrator.Kubernetes':
if 'tokens' not in source.node_template.attributes:
tokens = []
source.node_template.attributes['tokens'] = tokens
else:
tokens = source.node_template.attributes['tokens']
k8s_secrets = None
k8s_services = None
for task_output_key in task_outputs:
task_output = task_outputs[task_output_key]
if not k8s_secrets:
k8s_secrets = self.parse_ansible_var('k8s_secrets',task_output)
if not k8s_services:
k8s_services = self.parse_ansible_var('k8s_services', task_output)
if k8s_services and k8s_secrets:
break
source = self.set_kubernetes_attributes(source=source,task_outputs=task_outputs)
lst = list(nodes_pair)
lst[1] = source
nodes_pair = tuple(lst)
return nodes_pair
credential = {'token_type':'k8s_dashboard_token'}
credential['token'] = self.get_dashboard_token(k8s_secrets)
tokens.append(credential)
def parse_ansible_var(self,var_name, output_array):
index = 0
......@@ -85,20 +73,82 @@ class DeployService:
end_index = -1
for out in output_array:
index += 1
if 'TASK' in out or 'PLAY RECAP' in out:
if 'TASK' in out.output or 'PLAY RECAP' in out.output:
if start_index > -1:
end_index = index - 1
break
if '"' + var_name + '":' in out:
if start_index <=-1 and '"' + var_name + '":' in out.output:
start_index = index - 1
if start_index <= -1:
return None
ansible_var = output_array[start_index:end_index]
json_ansible_var = '{'
for item in ansible_var:
json_ansible_var = json_ansible_var + item
json_ansible_var = json_ansible_var + item.output
logger.info('found '+var_name +': '+ str(json_ansible_var))
return json.loads(json_ansible_var)
def get_dashboard_token(self, k8s_secrets):
print(k8s_secrets)
return None
\ No newline at end of file
def get_dashboard_token(self, k8s_dashboard_token):
k8s_dashboard_token = k8s_dashboard_token['k8s_dashboard_token']
# if 'resources' in k8s_secrets:
# return self.get_secret_from_k8s_info(k8s_secrets)
if 'stdout' in k8s_dashboard_token:
return self.get_secret_from_stdout(k8s_dashboard_token)
def get_service_port(self, k8s_services, service_name,port_type):
resources = k8s_services['k8s_services']['resources']
for resource in resources:
name = resource['metadata']['name']
if name == service_name:
ports = resource['spec']['ports']
for port in ports:
if port_type in port:
return port[port_type]
return None
def get_secret_from_k8s_info(self, k8s_secrets):
resources = k8s_secrets['resources']
for resource in resources:
metadata = resource['metadata']
if 'admin-user-token' in metadata['name']:
dashboard_token = resource['data']['token']
logger.info('found dashboard_token: ' + str(dashboard_token))
return resource['data']['token']
return None
def get_secret_from_stdout(self, k8s_dashboard_token):
return k8s_dashboard_token['stdout'].replace('token: ', '')
def set_kubernetes_attributes(self, source=None,task_outputs=None):
attributes = source.node_template.attributes
if 'tokens' not in attributes:
tokens = []
attributes['tokens'] = tokens
else:
tokens = attributes['tokens']
if 'dashboard_url' not in source.node_template.attributes:
dashboard_url = ''
attributes['dashboard_url'] = tokens
else:
dashboard_url = attributes['dashboard_url']
k8s_dashboard_token = None
k8s_services = None
for task_output_key in task_outputs:
task_output = task_outputs[task_output_key]
if not k8s_dashboard_token:
k8s_dashboard_token = self.parse_ansible_var('k8s_dashboard_token', task_output)
if not k8s_services:
k8s_services = self.parse_ansible_var('k8s_services', task_output)
if k8s_services and k8s_dashboard_token:
credential = {'token_type': 'k8s_dashboard_token'}
credential['token'] = self.get_dashboard_token(k8s_dashboard_token)
tokens.append(credential)
service_port = self.get_service_port(k8s_services, 'kubernetes-dashboard', 'nodePort')
dashboard_url = 'https://' + self.master_ip + ':' + str(service_port)
attributes['dashboard_url'] = dashboard_url
logger.info('source.node_template.attributes: ' + str(attributes))
return source
deployer/service/tosca_helper.py
View file @ 8e9dc5fb
......@@ -28,7 +28,6 @@ class ToscaHelper:
return api
def get_interface_types(target):
print(target.node_template.interfaces)
interface_types = []
for interface in target.node_template.interfaces:
interface_types.append(interface)
......@@ -73,6 +72,13 @@ class ToscaHelper:
def get_vms(self):
return self.tosca_client.get_node_templates(self.doc_id, type_name='tosca.nodes.QC.VM.Compute')
def set_node(self, updated_node, tosca_template_dict):
node_templates = tosca_template_dict['topology_template']['node_templates']
for node_name in node_templates:
if node_name == updated_node.name:
node_templates[node_name] = updated_node.node_template
return tosca_template_dict
def get_interface_types(node):
interface_type_names = []
......
deployer/test/test_deployer.py
View file @ 8e9dc5fb
......@@ -52,8 +52,9 @@ class TestDeployer(unittest.TestCase):
self.assertIsNotNone(nodes_pairs)
username = 'admin'
deployService = DeployService(polemarch_base_url=polemarch_base_url,polemarch_username=username,polemarch_password='admin',
semaphore_base_url=semaphore_base_url,semaphore_username=username,semaphore_password='password')
deployService = DeployService(polemarch_username=username,polemarch_password='admin',
semaphore_base_url=semaphore_base_url,semaphore_username=username,
semaphore_password='password')
for node_pair in nodes_pairs:
deployService.deploy(node_pair)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment