Commit f93ba4c8 authored by Spiros Koulouzis's avatar Spiros Koulouzis

added retry failed anible tasks and mammoth vm

parent b35cbba5
......@@ -116,7 +116,7 @@
<tbody data-link="row" class="rowlink">
<tr class="clickable-row" data-href="drip-api-json-client.jar">
<td><span class="downloadfile-name">drip-api-json-client.jar</span></td>
<td><span class="downloadfile-size">18.03K</span></td>
<td><span class="downloadfile-size">18.04K</span></td>
<td><span class="downloadfile-description">The binaries for the Java JSON client library.</span></td>
</tr>
<tr class="clickable-row" data-href="drip-api-json-client-json-sources.jar">
......@@ -149,7 +149,7 @@
</tr>
<tr class="clickable-row" data-href="drip-api-xml-client-xml-sources.jar">
<td><span class="downloadfile-name">drip-api-xml-client-xml-sources.jar</span></td>
<td><span class="downloadfile-size">23.28K</span></td>
<td><span class="downloadfile-size">23.27K</span></td>
<td><span class="downloadfile-description">The sources for the Java XML client library.</span></td>
</tr>
</tbody>
......@@ -184,7 +184,7 @@
<tbody data-link="row" class="rowlink">
<tr class="clickable-row" data-href="drip-api-js.zip">
<td><span class="downloadfile-name">drip-api-js.zip</span></td>
<td><span class="downloadfile-size">5.92K</span></td>
<td><span class="downloadfile-size">5.91K</span></td>
<td><span class="downloadfile-description">&nbsp;</span></td>
</tr>
</tbody>
......@@ -213,7 +213,7 @@
<tbody data-link="row" class="rowlink">
<tr class="clickable-row" data-href="drip-api-php.zip">
<td><span class="downloadfile-name">drip-api-php.zip</span></td>
<td><span class="downloadfile-size">6.67K</span></td>
<td><span class="downloadfile-size">6.64K</span></td>
<td><span class="downloadfile-description">&nbsp;</span></td>
</tr>
</tbody>
......
This diff is collapsed.
......@@ -81,25 +81,25 @@ class</p>
</thead>
<tbody>
<tr>
<td> <span class="property-name">end</span>
<td> <span class="property-name">cloudDeploymentDomain</span>
</td>
<td> <span class="datatype-reference">number
<td> <span class="datatype-reference">string
</span>
</td>
<td> <span class="property-description"></span>
</td>
</tr>
<tr>
<td> <span class="property-name">vmType</span>
<td> <span class="property-name">start</span>
</td>
<td> <span class="datatype-reference">string
<td> <span class="datatype-reference">number
</span>
</td>
<td> <span class="property-description"></span>
</td>
</tr>
<tr>
<td> <span class="property-name">host</span>
<td> <span class="property-name">vmType</span>
</td>
<td> <span class="datatype-reference">string
</span>
......@@ -117,7 +117,7 @@ class</p>
</td>
</tr>
<tr>
<td> <span class="property-name">cloudDeploymentDomain</span>
<td> <span class="property-name">host</span>
</td>
<td> <span class="datatype-reference">string
</span>
......@@ -126,7 +126,7 @@ class</p>
</td>
</tr>
<tr>
<td> <span class="property-name">start</span>
<td> <span class="property-name">end</span>
</td>
<td> <span class="datatype-reference">number
</span>
......@@ -167,12 +167,12 @@ No need to set during a POST</span></td>
<p class="lead">Example</p>
<pre class="prettyprint language-js example">{
&quot;end&quot; : 12345,
&quot;vmType&quot; : &quot;...&quot;,
&quot;host&quot; : &quot;147.228.242.58&quot;,
&quot;delta&quot; : 12345,
&quot;cloudDeploymentDomain&quot; : &quot;us-east-1&quot;,
&quot;start&quot; : 12345,
&quot;vmType&quot; : &quot;...&quot;,
&quot;delta&quot; : 12345,
&quot;host&quot; : &quot;147.228.242.58&quot;,
&quot;end&quot; : 12345,
&quot;owner&quot; : &quot;user1&quot;,
&quot;id&quot; : &quot;58e3946e0fb4f562d84ba1ad&quot;,
&quot;timestamp&quot; : 1499793079011
......
......@@ -69,7 +69,7 @@ class</p>
<dl class="dl-horizontal">
<dt>Subtypes</dt>
<dd><a href="json_KeyPair.html">KeyPair</a>, <a href="json_ScaleRequest.html">ScaleRequest</a>, <a href="json_AnsibleOutput.html">AnsibleOutput</a>, <a href="json_CloudCredentials.html">CloudCredentials</a>, <a href="json_BenchmarkResult.html">BenchmarkResult</a>, <a href="json_DeployResponse.html">DeployResponse</a>, <a href="json_DRIPLogRecord.html">DRIPLogRecord</a>, <a href="json_ProvisionRequest.html">ProvisionRequest</a>, <a href="json_Script.html">Script</a>, <a href="json_KeyValueHolder.html">KeyValueHolder</a>, <a href="json_DeployRequest.html">DeployRequest</a>, <a href="json_MonitorringMessage.html">MonitorringMessage</a>, <a href="json_ProvisionResponse.html">ProvisionResponse</a></dd>
<dd><a href="json_KeyPair.html">KeyPair</a>, <a href="json_AnsibleOutput.html">AnsibleOutput</a>, <a href="json_ScaleRequest.html">ScaleRequest</a>, <a href="json_BenchmarkResult.html">BenchmarkResult</a>, <a href="json_CloudCredentials.html">CloudCredentials</a>, <a href="json_DeployResponse.html">DeployResponse</a>, <a href="json_DRIPLogRecord.html">DRIPLogRecord</a>, <a href="json_ProvisionRequest.html">ProvisionRequest</a>, <a href="json_Script.html">Script</a>, <a href="json_KeyValueHolder.html">KeyValueHolder</a>, <a href="json_DeployRequest.html">DeployRequest</a>, <a href="json_MonitorringMessage.html">MonitorringMessage</a>, <a href="json_ProvisionResponse.html">ProvisionResponse</a></dd>
</dl>
<table class="table datatype-properties">
......
......@@ -825,13 +825,13 @@ class]]>
</xs:element>
<xs:element name="vmType" type="xs:string" minOccurs="0">
</xs:element>
<xs:element name="cloudDeploymentDomain" type="xs:string" minOccurs="0">
</xs:element>
<xs:element name="end" type="xs:dateTime" minOccurs="0">
</xs:element>
<xs:element name="delta" type="xs:dateTime" minOccurs="0">
</xs:element>
<xs:element name="host" type="xs:string" minOccurs="0">
</xs:element>
<xs:element name="delta" type="xs:dateTime" minOccurs="0">
<xs:element name="cloudDeploymentDomain" type="xs:string" minOccurs="0">
</xs:element>
</xs:sequence>
</xs:extension>
......
......@@ -144,12 +144,12 @@ Content-Type: application/json
<code class="prettyprint language-js">
[ {
&quot;end&quot; : 12345,
&quot;vmType&quot; : &quot;...&quot;,
&quot;host&quot; : &quot;147.228.242.58&quot;,
&quot;delta&quot; : 12345,
&quot;cloudDeploymentDomain&quot; : &quot;us-east-1&quot;,
&quot;start&quot; : 12345,
&quot;vmType&quot; : &quot;...&quot;,
&quot;delta&quot; : 12345,
&quot;host&quot; : &quot;147.228.242.58&quot;,
&quot;end&quot; : 12345,
&quot;owner&quot; : &quot;user1&quot;,
&quot;id&quot; : &quot;58e3946e0fb4f562d84ba1ad&quot;,
&quot;timestamp&quot; : 1499793079011
......@@ -514,12 +514,12 @@ Content-Type: application/json
<code class="prettyprint language-js">
{
&quot;end&quot; : 12345,
&quot;vmType&quot; : &quot;...&quot;,
&quot;host&quot; : &quot;147.228.242.58&quot;,
&quot;delta&quot; : 12345,
&quot;cloudDeploymentDomain&quot; : &quot;us-east-1&quot;,
&quot;start&quot; : 12345,
&quot;vmType&quot; : &quot;...&quot;,
&quot;delta&quot; : 12345,
&quot;host&quot; : &quot;147.228.242.58&quot;,
&quot;end&quot; : 12345,
&quot;owner&quot; : &quot;user1&quot;,
&quot;id&quot; : &quot;58e3946e0fb4f562d84ba1ad&quot;,
&quot;timestamp&quot; : 1499793079011
......
......@@ -222,10 +222,10 @@ It is created automatically. No need to set during a POST</span></td>
<pre class="prettyprint language-xml example">&lt;b-----&gt;
&lt;start&gt;...&lt;/start&gt;
&lt;vmType&gt;...&lt;/vmType&gt;
&lt;cloudDeploymentDomain&gt;us-east-1&lt;/cloudDeploymentDomain&gt;
&lt;end&gt;...&lt;/end&gt;
&lt;host&gt;147.228.242.58&lt;/host&gt;
&lt;delta&gt;...&lt;/delta&gt;
&lt;host&gt;147.228.242.58&lt;/host&gt;
&lt;cloudDeploymentDomain&gt;us-east-1&lt;/cloudDeploymentDomain&gt;
&lt;owner&gt;user1&lt;/owner&gt;
&lt;/b-----&gt;
</pre>
......
......@@ -116,7 +116,7 @@
<tbody data-link="row" class="rowlink">
<tr class="clickable-row" data-href="drip-api-json-client.jar">
<td><span class="downloadfile-name">drip-api-json-client.jar</span></td>
<td><span class="downloadfile-size">18.03K</span></td>
<td><span class="downloadfile-size">18.04K</span></td>
<td><span class="downloadfile-description">The binaries for the Java JSON client library.</span></td>
</tr>
<tr class="clickable-row" data-href="drip-api-json-client-json-sources.jar">
......@@ -149,7 +149,7 @@
</tr>
<tr class="clickable-row" data-href="drip-api-xml-client-xml-sources.jar">
<td><span class="downloadfile-name">drip-api-xml-client-xml-sources.jar</span></td>
<td><span class="downloadfile-size">23.28K</span></td>
<td><span class="downloadfile-size">23.27K</span></td>
<td><span class="downloadfile-description">The sources for the Java XML client library.</span></td>
</tr>
</tbody>
......@@ -184,7 +184,7 @@
<tbody data-link="row" class="rowlink">
<tr class="clickable-row" data-href="drip-api-js.zip">
<td><span class="downloadfile-name">drip-api-js.zip</span></td>
<td><span class="downloadfile-size">5.92K</span></td>
<td><span class="downloadfile-size">5.91K</span></td>
<td><span class="downloadfile-description">&nbsp;</span></td>
</tr>
</tbody>
......@@ -213,7 +213,7 @@
<tbody data-link="row" class="rowlink">
<tr class="clickable-row" data-href="drip-api-php.zip">
<td><span class="downloadfile-name">drip-api-php.zip</span></td>
<td><span class="downloadfile-size">6.67K</span></td>
<td><span class="downloadfile-size">6.64K</span></td>
<td><span class="downloadfile-description">&nbsp;</span></td>
</tr>
</tbody>
......
This diff is collapsed.
......@@ -81,7 +81,7 @@ class</p>
</thead>
<tbody>
<tr>
<td> <span class="property-name">vmType</span>
<td> <span class="property-name">host</span>
</td>
<td> <span class="datatype-reference">string
</span>
......@@ -99,7 +99,7 @@ class</p>
</td>
</tr>
<tr>
<td> <span class="property-name">delta</span>
<td> <span class="property-name">end</span>
</td>
<td> <span class="datatype-reference">number
</span>
......@@ -117,7 +117,7 @@ class</p>
</td>
</tr>
<tr>
<td> <span class="property-name">end</span>
<td> <span class="property-name">delta</span>
</td>
<td> <span class="datatype-reference">number
</span>
......@@ -126,7 +126,7 @@ class</p>
</td>
</tr>
<tr>
<td> <span class="property-name">host</span>
<td> <span class="property-name">vmType</span>
</td>
<td> <span class="datatype-reference">string
</span>
......@@ -167,12 +167,12 @@ No need to set during a POST</span></td>
<p class="lead">Example</p>
<pre class="prettyprint language-js example">{
&quot;vmType&quot; : &quot;...&quot;,
&quot;host&quot; : &quot;147.228.242.58&quot;,
&quot;start&quot; : 12345,
&quot;delta&quot; : 12345,
&quot;cloudDeploymentDomain&quot; : &quot;us-east-1&quot;,
&quot;end&quot; : 12345,
&quot;host&quot; : &quot;147.228.242.58&quot;,
&quot;cloudDeploymentDomain&quot; : &quot;us-east-1&quot;,
&quot;delta&quot; : 12345,
&quot;vmType&quot; : &quot;...&quot;,
&quot;owner&quot; : &quot;user1&quot;,
&quot;id&quot; : &quot;58e3946e0fb4f562d84ba1ad&quot;,
&quot;timestamp&quot; : 1499793079011
......
......@@ -821,16 +821,16 @@ class]]>
<xs:complexContent>
<xs:extension base="ownedObject">
<xs:sequence>
<xs:element name="delta" type="xs:dateTime" minOccurs="0">
</xs:element>
<xs:element name="cloudDeploymentDomain" type="xs:string" minOccurs="0">
</xs:element>
<xs:element name="vmType" type="xs:string" minOccurs="0">
</xs:element>
<xs:element name="host" type="xs:string" minOccurs="0">
</xs:element>
<xs:element name="start" type="xs:dateTime" minOccurs="0">
</xs:element>
<xs:element name="delta" type="xs:dateTime" minOccurs="0">
</xs:element>
<xs:element name="vmType" type="xs:string" minOccurs="0">
</xs:element>
<xs:element name="end" type="xs:dateTime" minOccurs="0">
</xs:element>
</xs:sequence>
......
......@@ -144,12 +144,12 @@ Content-Type: application/json
<code class="prettyprint language-js">
[ {
&quot;vmType&quot; : &quot;...&quot;,
&quot;host&quot; : &quot;147.228.242.58&quot;,
&quot;start&quot; : 12345,
&quot;delta&quot; : 12345,
&quot;cloudDeploymentDomain&quot; : &quot;us-east-1&quot;,
&quot;end&quot; : 12345,
&quot;host&quot; : &quot;147.228.242.58&quot;,
&quot;cloudDeploymentDomain&quot; : &quot;us-east-1&quot;,
&quot;delta&quot; : 12345,
&quot;vmType&quot; : &quot;...&quot;,
&quot;owner&quot; : &quot;user1&quot;,
&quot;id&quot; : &quot;58e3946e0fb4f562d84ba1ad&quot;,
&quot;timestamp&quot; : 1499793079011
......@@ -514,12 +514,12 @@ Content-Type: application/json
<code class="prettyprint language-js">
{
&quot;vmType&quot; : &quot;...&quot;,
&quot;host&quot; : &quot;147.228.242.58&quot;,
&quot;start&quot; : 12345,
&quot;delta&quot; : 12345,
&quot;cloudDeploymentDomain&quot; : &quot;us-east-1&quot;,
&quot;end&quot; : 12345,
&quot;host&quot; : &quot;147.228.242.58&quot;,
&quot;cloudDeploymentDomain&quot; : &quot;us-east-1&quot;,
&quot;delta&quot; : 12345,
&quot;vmType&quot; : &quot;...&quot;,
&quot;owner&quot; : &quot;user1&quot;,
&quot;id&quot; : &quot;58e3946e0fb4f562d84ba1ad&quot;,
&quot;timestamp&quot; : 1499793079011
......
......@@ -220,11 +220,11 @@ It is created automatically. No need to set during a POST</span></td>
<p class="lead">Example</p>
<pre class="prettyprint language-xml example">&lt;b-----&gt;
&lt;delta&gt;...&lt;/delta&gt;
&lt;cloudDeploymentDomain&gt;us-east-1&lt;/cloudDeploymentDomain&gt;
&lt;vmType&gt;...&lt;/vmType&gt;
&lt;host&gt;147.228.242.58&lt;/host&gt;
&lt;start&gt;...&lt;/start&gt;
&lt;delta&gt;...&lt;/delta&gt;
&lt;vmType&gt;...&lt;/vmType&gt;
&lt;end&gt;...&lt;/end&gt;
&lt;owner&gt;user1&lt;/owner&gt;
&lt;/b-----&gt;
......
......@@ -55,7 +55,9 @@ if not getattr(logger, 'handler_set', None):
retry=0
#cwd = os.getcwd()
falied_playbook='/tmp/falied_playbook.yml'
def install_prerequisites(vm,return_dict):
try:
......@@ -87,42 +89,52 @@ def install_prerequisites(vm,return_dict):
return "ERROR:"+vm.ip+" "+str(e)
ssh.close()
return_dict[vm.ip] = "SUCCESS"
retry = 0
return "SUCCESS"
def run_faied(failed_tasks,inventory,variable_manager,loader,options,passwords,results_callback,playbook_path):
def create_faied_playbooks(failed_tasks,inventory,variable_manager,loader,options,passwords,results_callback,playbook_path):
tasks = []
hosts = []
#tqm = TaskQueueManager(inventory=inventory, variable_manager=variable_manager, loader=loader, options=options, passwords=passwords)
#yml_plays = {}
#with open(playbook_path) as stream:
#yml_plays = yaml.load(stream)
#failed_yml = {}
#retry_task = []
#hosts = []
#for failed_task in failed_tasks:
#name = failed_task._task.get_name()
#host = failed_task._host.get_name()
plays = {}
with open(playbook_path) as stream:
plays = yaml.load(stream)
failed_plays = []
for failed_task in failed_tasks:
failed_play = {}
retry_task = []
hosts = ""
if isinstance(failed_task, ansible.parsing.yaml.objects.AnsibleUnicode) or isinstance(failed_task, unicode):
task_name = str(failed_task)
host = str(failed_task)
else:
task_name = str(failed_task._task.get_name())
host = str(failed_task._host.get_name())
#for play in yml_plays:
#for task in play['tasks']:
#if name in task['name']:
#retry_task.append(task)
#hosts.append(host)
#failed_yml['hosts'] = hosts
#failed_yml['tasks'] = retry_task
#with open('/tmp/failed.yml', 'w') as outfile:
#yaml.dump(failed_yml, outfile, default_flow_style=False)
#play = Play().load('/tmp/failed.yml', variable_manager=variable_manager, loader=loader)
#res = tqm.run(play=play)
for play in plays:
for task in play['tasks']:
if 'name' in task and task['name'] == task_name:
retry_task.append(task)
logger.warning("Faield task: \'"+task_name+ "\' In host: "+ host)
if host not in hosts:
hosts +=host+","
break
elif task_name in task:
retry_task.append(task)
logger.warning("Faield task: \'"+task_name+ "\' In host: "+ host)
if host not in hosts:
hosts +=host+","
break
failed_play['hosts'] = hosts
failed_play['tasks'] = retry_task
failed_plays.append(failed_play)
with open(falied_playbook, 'w') as outfile:
yaml.dump(failed_plays, outfile)
def execute_playbook(hosts, playbook_path,user,ssh_key_file,extra_vars,passwords):
if not os.path.exists(playbook_path):
......@@ -160,44 +172,38 @@ def execute_playbook(hosts, playbook_path,user,ssh_key_file,extra_vars,passwords
results = pbex.run()
ok = results_callback.host_ok
answer = []
failed_tasks = []
for res in ok:
#failed_tasks.append(res['result'])
for res in ok:
resp = json.dumps({"host":res['ip'], "result":res['result']._result,"task":res['task']})
#logger.info(resp)
logger.info(resp)
answer.append({"host":res['ip'], "result":res['result']._result,"task":res['task']})
unreachable = results_callback.host_unreachable
for res in unreachable:
#failed_tasks.append(res['task'])
failed_tasks.append(res['task'])
resp = json.dumps({"host":res['ip'], "result":res['result']._result,"task":res['task']})
logger.info(resp)
answer.append({"host":res['ip'], "result":res['result']._result,"task":res['task']})
host_failed = results_callback.host_failed
for res in host_failed:
#failed_tasks.append(res['result'])
resp = json.dumps({"host":res['ip'], "result":res['result']._result, "task":res['task']})
#logger.info(resp)
logger.info(resp)
answer.append({"host":res['ip'], "result":res['result']._result,"task":res['task']})
if failed_tasks:
run_faied(failed_tasks,inventory,variable_manager,loader,options,passwords,results_callback,playbook_path)
create_faied_playbooks(failed_tasks,inventory,variable_manager,loader,options,passwords,results_callback,playbook_path)
return json.dumps(answer)
def run(vm_list,playbook_path,rabbitmq_host,owner):
#Create /playbook.retry
hosts=""
ssh_key_file=""
rabbit = DRIPLoggingHandler(host=rabbitmq_host, port=5672,user=owner)
......@@ -207,6 +213,8 @@ def run(vm_list,playbook_path,rabbitmq_host,owner):
return_dict = manager.dict()
jobs = []
if os.path.exists(falied_playbook):
os.remove(falied_playbook)
for vm in vm_list:
#ret = install_prerequisites(vm,return_dict)
......@@ -224,4 +232,13 @@ def run(vm_list,playbook_path,rabbitmq_host,owner):
extra_vars = {}
passwords = {}
logger.info("Executing playbook: " + (playbook_path))
return execute_playbook(hosts,playbook_path,user,ssh_key_file,extra_vars,passwords)
\ No newline at end of file
retry = 0
res = execute_playbook(hosts,playbook_path,user,ssh_key_file,extra_vars,passwords)
while os.path.exists(falied_playbook) and retry < 1:
retry+=1
logger.warning("Some tasks faield retrying: "+str(retry))
res = execute_playbook(hosts,playbook_path,user,ssh_key_file,extra_vars,passwords)
return res
\ No newline at end of file
#!/usr/bin/env python
import pika
import json
import os
import time
from vm_info import VmInfo
import docker_kubernetes
import docker_engine
import docker_swarm
import docker_compose
import docker_service
import docker_check
import control_agent
import ansible_playbook
import sys, argparse
from threading import Thread
from time import sleep
import os.path
import logging
from os.path import expanduser
hosts="localhost,"
home = expanduser("~")
playbook_path=home+"/workspace/DRIP/drip-deployer/deployer_files/1513695133139/playbook.yml"
user="vm_user"
ssh_key_file=home+"/workspace/DRIP/drip-deployer/deployer_files/1513695133139/1.txt"
extra_vars = {}
passwords = {}
ansible_playbook.execute_playbook(hosts,playbook_path,user,ssh_key_file,extra_vars,passwords)
\ No newline at end of file
......@@ -4,4 +4,4 @@ CESNET&&https://carach5.ics.muni.cz:11443&&ubuntu 16.04&&medium&&http://fedcloud
CESNET&&https://carach5.ics.muni.cz:11443&&ubuntu 16.04&&0#mem_large&&http://fedcloud.egi.eu/occi/compute/flavour/1.0#mem_large&&http://occi.carach5.ics.muni.cz/occi/infrastructure/os_tpl#uuid_8df7ba00_8467_57aa_bf1e_05754a2a73bf_warg_default_shared_219&&ubuntu
CESNET&&https://carach5.ics.muni.cz:11443&&ubuntu 16.04&&extra_large&&http://schemas.fedcloud.egi.eu/occi/infrastructure/resource_tpl#extra_large&&http://occi.carach5.ics.muni.cz/occi/infrastructure/os_tpl#uuid_8df7ba00_8467_57aa_bf1e_05754a2a73bf_warg_default_shared_219&&ubuntu
CESNET&&https://carach5.ics.muni.cz:11443&&ubuntu 16.04&&small&&http://fedcloud.egi.eu/occi/compute/flavour/1.0#small&&http://occi.carach5.ics.muni.cz/occi/infrastructure/os_tpl#uuid_8df7ba00_8467_57aa_bf1e_05754a2a73bf_warg_default_shared_219&&ubuntu
CESNET&&https://carach5.ics.muni.cz:11443&&ubuntu 16.04&&0#sym_mammoth&&http://fedcloud.egi.eu/occi/compute/flavour/1.0#sym_mammoth&&http://occi.carach5.ics.muni.cz/occi/infrastructure/os_tpl#uuid_8df7ba00_8467_57aa_bf1e_05754a2a73bf_warg_default_shared_219&&ubuntu
\ No newline at end of file
CESNET&&https://carach5.ics.muni.cz:11443&&ubuntu 16.04&&mammoth&&http://schemas.fedcloud.egi.eu/occi/infrastructure/resource_tpl#mammoth&&http://occi.carach5.ics.muni.cz/occi/infrastructure/os_tpl#uuid_8df7ba00_8467_57aa_bf1e_05754a2a73bf_warg_default_shared_219&&ubuntu
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment