Skip to content

C
CONF
Commit 21ce2a62 authored by Alfonso Orta's avatar Alfonso Orta
Browse files
Options

Merge branch 'develop' into 'staging' 

Develop

See merge request !3
parents 0202be19 ea18c6db
Show whitespace changes
Inline Side-by-side
Showing with 857 additions and 315 deletions
bin/docker_push.sh
View file @ 21ce2a62
#!/bin/bash #!/bin/bash
docker push qcdis/manager:3.0.0 docker push qcdis/manager
docker push qcdis/provisioner:3.0.0 docker push qcdis/provisioner
# docker push qcdis/sure-tosca:3.0.0 docker push qcdis/sure-tosca
# docker push qcdis/planner:3.0.0 docker push qcdis/planner
# docker push qcdis/deployer:3.0.0 docker push qcdis/deployer
bin/dockerize.sh
View file @ 21ce2a62
...@@ -10,21 +10,21 @@ cd provisioner && mvn -Dmaven.test.skip=true dockerfile:build ...@@ -10,21 +10,21 @@ cd provisioner && mvn -Dmaven.test.skip=true dockerfile:build
[ $status -eq 0 ] && echo "build successful" || exit -1 [ $status -eq 0 ] && echo "build successful" || exit -1
cd ../ cd ../
echo "----------Building sure_tosca-flask-server Docker--------------" echo "----------Building sure_tosca-flask-server Docker--------------"
cd sure_tosca-flask-server && docker build -t sure-tosca:3.0.0 . cd sure_tosca-flask-server && docker build -t sure-tosca .
[ $status -eq 0 ] && echo "build successful" || exit -1 [ $status -eq 0 ] && echo "build successful" || exit -1
echo "----------Building planner Docker--------------" echo "----------Building planner Docker--------------"
cd ../ cd ../
cd planner && docker build -t planner:3.0.0 . cd planner && docker build -t planner .
[ $status -eq 0 ] && echo "build successful" || exit -1 [ $status -eq 0 ] && echo "build successful" || exit -1
echo "----------Building deployer Docker--------------" echo "----------Building deployer Docker--------------"
cd ../ cd ../
cd deployer && docker build -t deployer:3.0.0 . cd deployer && docker build -t deployer .
[ $status -eq 0 ] && echo "build successful" || exit -1 [ $status -eq 0 ] && echo "build successful" || exit -1
docker tag sure-tosca:3.0.0 qcdis/sure-tosca:3.0.0 docker tag sure-tosca qcdis/sure-tosca
docker tag planner:3.0.0 qcdis/planner:3.0.0 docker tag planner qcdis/planner
docker tag deployer:3.0.0 qcdis/deployer:3.0.0 docker tag deployer qcdis/deployer
docker tag manager:3.0.0 qcdis/manager:3.0.0 docker tag manager:3.0.0 qcdis/manager
docker tag provisioner:3.0.0 qcdis/provisioner:3.0.0 docker tag provisioner:3.0.0 qcdis/provisioner
commons/src/main/java/nl/uva/sne/drip/commons/utils/Converter.java
View file @ 21ce2a62
...@@ -21,6 +21,7 @@ import java.io.FileNotFoundException; ...@@ -21,6 +21,7 @@ import java.io.FileNotFoundException;
import java.io.FileOutputStream; import java.io.FileOutputStream;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.io.UnsupportedEncodingException;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.nio.file.FileSystem; import java.nio.file.FileSystem;
import java.nio.file.FileSystems; import java.nio.file.FileSystems;
...@@ -30,16 +31,26 @@ import java.nio.file.Path; ...@@ -30,16 +31,26 @@ import java.nio.file.Path;
import java.nio.file.Paths; import java.nio.file.Paths;
import java.nio.file.SimpleFileVisitor; import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes; import java.nio.file.attribute.BasicFileAttributes;
import java.security.InvalidKeyException;
import java.security.Key;
import java.security.MessageDigest; import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException; import java.security.NoSuchAlgorithmException;
import java.util.Arrays;
import java.util.Base64; import java.util.Base64;
import java.util.Enumeration; import java.util.Enumeration;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set;
import java.util.zip.ZipEntry; import java.util.zip.ZipEntry;
import java.util.zip.ZipFile; import java.util.zip.ZipFile;
import java.util.zip.ZipOutputStream; import java.util.zip.ZipOutputStream;
import javax.crypto.BadPaddingException;
import javax.crypto.Cipher;
import javax.crypto.IllegalBlockSizeException;
import javax.crypto.NoSuchPaddingException;
import javax.crypto.spec.SecretKeySpec;
import nl.uva.sne.drip.model.tosca.Credential;
import org.json.JSONException; import org.json.JSONException;
import org.json.JSONObject; import org.json.JSONObject;
import org.springframework.web.multipart.MultipartFile; import org.springframework.web.multipart.MultipartFile;
...@@ -157,4 +168,62 @@ public class Converter { ...@@ -157,4 +168,62 @@ public class Converter {
} }
} }
public static String decryptString(String contents, String secret) throws UnsupportedEncodingException, NoSuchAlgorithmException, NoSuchPaddingException, InvalidKeyException, IllegalBlockSizeException, BadPaddingException {
SecretKeySpec secretKey = getsecretKey(secret);
Cipher cipher = Cipher.getInstance("AES/ECB/PKCS5PADDING");
cipher.init(Cipher.DECRYPT_MODE, secretKey);
return new String(cipher.doFinal(Base64.getDecoder().decode(contents)));
}
public static String encryptString(String contents, String secret) throws UnsupportedEncodingException, NoSuchAlgorithmException, NoSuchPaddingException, InvalidKeyException, IllegalBlockSizeException, BadPaddingException {
SecretKeySpec secretKey = getsecretKey(secret);
Cipher cipher = Cipher.getInstance("AES/ECB/PKCS5Padding");
cipher.init(Cipher.ENCRYPT_MODE, secretKey);
return Base64.getEncoder().encodeToString(cipher.doFinal(contents.getBytes("UTF-8")));
}
private static SecretKeySpec getsecretKey(String myKey) throws UnsupportedEncodingException, NoSuchAlgorithmException {
MessageDigest sha;
byte[] key = myKey.getBytes("UTF-8");
sha = MessageDigest.getInstance("SHA-1");
key = sha.digest(key);
key = Arrays.copyOf(key, 16);
return new SecretKeySpec(key, "AES");
}
public static Credential encryptCredential(Credential credential, String credentialSecret) throws UnsupportedEncodingException, NoSuchAlgorithmException, NoSuchPaddingException, InvalidKeyException, IllegalBlockSizeException, BadPaddingException {
Map<String, String> credKeys = credential.getKeys();
Set<String> keySet = credKeys.keySet();
for (String key : keySet) {
String credKey = credKeys.get(key);
if (credKey != null) {
credKeys.put(key, encryptString(credKey, credentialSecret));
}
}
String token = credential.getToken();
if (token != null) {
credential.setToken(encryptString(token, credentialSecret));
}
return credential;
}
public static Credential dencryptCredential(Credential credential, String credentialSecret) throws UnsupportedEncodingException, NoSuchAlgorithmException, NoSuchPaddingException, InvalidKeyException, IllegalBlockSizeException, BadPaddingException {
Map<String, String> credKeys = credential.getKeys();
Set<String> keySet = credKeys.keySet();
for (String key : keySet) {
String credKey = credKeys.get(key);
if (credKey != null) {
credKeys.put(key, decryptString(credKey, credentialSecret));
}
}
String token = credential.getToken();
if (token != null) {
credential.setToken(decryptString(token, credentialSecret));
}
return credential;
}
} }
commons/src/main/resources/application.properties deleted 100644 → 0
View file @ 0202be19
message.broker.host=127.0.0.1
message.broker.username=guest
message.broker.password=guest
message.broker.queue.provisioner=provisioner
message.broker.queue.planner=planner
message.broker.queue.deployer=deployer
sure-tosca.base.path=http://localhost:8081/tosca-sure/1.0.0
commons/src/test/java/nl/uva/sne/drip/commons/utils/ConverterTest.java 0 → 100644
View file @ 21ce2a62
/*
* Copyright 2019 S. Koulouzis
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nl.uva.sne.drip.commons.utils;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import static org.junit.Assert.*;
/**
*
* @author S. Koulouzis
*/
public class ConverterTest {
public ConverterTest() {
}
@BeforeClass
public static void setUpClass() {
}
@AfterClass
public static void tearDownClass() {
}
@Before
public void setUp() {
}
@After
public void tearDown() {
}
/**
* Test of decryptString method, of class Converter.
*/
@Test
public void testEncryptDecryptString() throws Exception {
System.out.println("decryptString");
String contents = "this is very important information";
String secret = "top_secret";
String expResult = contents;
String enc = Converter.encryptString(contents, secret);
System.out.println("Encrypted String: " + enc);
String result = Converter.decryptString(enc, secret);
assertEquals(expResult, result);
}
}
deployer/Dockerfile
View file @ 21ce2a62
...@@ -28,4 +28,7 @@ RUN pip3 install --no-cache-dir -r requirements.txt ...@@ -28,4 +28,7 @@ RUN pip3 install --no-cache-dir -r requirements.txt
COPY . /usr/src/app COPY . /usr/src/app
CMD python3 __main__.py $RABBITMQ_HOST deployer CMD sed -i "s#http://127.0.0.1:8081/tosca-sure/1.0.0#$SURE_TOSCA_BASE_PATH#g" properties.ini && \
sed -i "s#http://127.0.0.1:3000/api#$SEMAPHORE_BASE_PATH#g" properties.ini && \
sed -i "s#host = 127.0.0.1#host = $RABBITMQ_HOST#g" properties.ini && \
python3 __main__.py $RABBITMQ_HOST deployer
deployer/__main__.py
View file @ 21ce2a62
...@@ -8,14 +8,16 @@ import os ...@@ -8,14 +8,16 @@ import os
import sys import sys
import tempfile import tempfile
import time import time
import traceback
from threading import Thread from threading import Thread
from time import sleep from time import sleep
from service.deploy_service import DeployService
from service.tosca_helper import ToscaHelper
import pika import pika
import sure_tosca_client
import yaml import yaml
from service.deploy_service import DeployService
from service.tosca_helper import ToscaHelper
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
done = False done = False
...@@ -85,9 +87,17 @@ def handle_delivery(message): ...@@ -85,9 +87,17 @@ def handle_delivery(message):
nodes_pairs = tosca_helper.get_deployment_node_pairs() nodes_pairs = tosca_helper.get_deployment_node_pairs()
deployService = DeployService(semaphore_base_url=semaphore_base_url, semaphore_username=semaphore_username, deployService = DeployService(semaphore_base_url=semaphore_base_url, semaphore_username=semaphore_username,
semaphore_password=semaphore_password) semaphore_password=semaphore_password, vms=tosca_helper.get_vms())
try:
for node_pair in nodes_pairs: for node_pair in nodes_pairs:
deployService.deploy(node_pair) updated_node = deployService.deploy(node_pair)
if isinstance(updated_node, list):
for node in updated_node:
tosca_template_dict = tosca_helper.set_node(node,tosca_template_dict)
logger.info("tosca_template_dict :" + json.dumps(tosca_template_dict))
else:
tosca_template_dict = tosca_helper.set_node(updated_node, tosca_template_dict)
logger.info("tosca_template_dict :" + json.dumps(tosca_template_dict))
response = {'toscaTemplate': tosca_template_dict} response = {'toscaTemplate': tosca_template_dict}
output_current_milli_time = int(round(time.time() * 1000)) output_current_milli_time = int(round(time.time() * 1000))
...@@ -95,6 +105,10 @@ def handle_delivery(message): ...@@ -95,6 +105,10 @@ def handle_delivery(message):
logger.info("Returning Deployment") logger.info("Returning Deployment")
logger.info("Output message:" + json.dumps(response)) logger.info("Output message:" + json.dumps(response))
return json.dumps(response) return json.dumps(response)
except Exception as e:
track = traceback.format_exc()
print(track)
raise
def threaded_function(args): def threaded_function(args):
......
deployer/build/lib/test/test_deployer.py
View file @ 21ce2a62
...@@ -39,7 +39,6 @@ class TestDeployer(unittest.TestCase): ...@@ -39,7 +39,6 @@ class TestDeployer(unittest.TestCase):
sure_tosca_base_url = 'http://127.0.0.1:8081/tosca-sure/1.0.0' sure_tosca_base_url = 'http://127.0.0.1:8081/tosca-sure/1.0.0'
polemarch_base_url='http://127.0.0.1:30001/api/v2'
semaphore_base_url = 'http://127.0.0.1:3000/api' semaphore_base_url = 'http://127.0.0.1:3000/api'
tosca_service_is_up = ToscaHelper.service_is_up(sure_tosca_base_url) tosca_service_is_up = ToscaHelper.service_is_up(sure_tosca_base_url)
semaphore_is_up = ToscaHelper.service_is_up(semaphore_base_url) semaphore_is_up = ToscaHelper.service_is_up(semaphore_base_url)
......
deployer/service/ansible_service.py
View file @ 21ce2a62
...@@ -6,7 +6,7 @@ import datetime ...@@ -6,7 +6,7 @@ import datetime
import yaml import yaml
from semaphore_client.semaphore_helper import SemaphoreHelper from semaphore_client.semaphore_helper import SemaphoreHelper
yaml.Dumper.ignore_aliases = lambda *args : True yaml.Dumper.ignore_aliases = lambda *args: True
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
if not getattr(logger, 'handler_set', None): if not getattr(logger, 'handler_set', None):
...@@ -20,21 +20,20 @@ if not getattr(logger, 'handler_set', None): ...@@ -20,21 +20,20 @@ if not getattr(logger, 'handler_set', None):
class AnsibleService: class AnsibleService:
def __init__(self, semaphore_base_url=None,semaphore_username=None,semaphore_password=None): def __init__(self, semaphore_base_url=None, semaphore_username=None, semaphore_password=None):
self.semaphore_base_url = semaphore_base_url self.semaphore_base_url = semaphore_base_url
self.semaphore_username = semaphore_username self.semaphore_username = semaphore_username
self.semaphore_password = semaphore_password self.semaphore_password = semaphore_password
self.semaphore_helper = SemaphoreHelper(self.semaphore_base_url, self.semaphore_username, self.semaphore_password) self.semaphore_helper = SemaphoreHelper(self.semaphore_base_url, self.semaphore_username,
self.semaphore_password)
self.repository_id = None self.repository_id = None
self.template_id = None self.template_id = None
def execute(self, nodes_pair, interface_type, vms, env_vars=None):
def execute(self,nodes_pair):
vms = nodes_pair[0]
application = nodes_pair[1] application = nodes_pair[1]
name = application.name name = application.name
desired_state = None desired_state = None
tasks_outputs = {}
interfaces = application.node_template.interfaces interfaces = application.node_template.interfaces
if 'current_state' in application.node_template.attributes: if 'current_state' in application.node_template.attributes:
current_state = application.node_template.attributes['current_state'] current_state = application.node_template.attributes['current_state']
...@@ -43,31 +42,45 @@ class AnsibleService: ...@@ -43,31 +42,45 @@ class AnsibleService:
if desired_state: if desired_state:
now = datetime.datetime.now() now = datetime.datetime.now()
project_id = self.semaphore_helper.create_project(application.name+'_'+str(now)) project_id = self.semaphore_helper.create_project(application.name + '_' + str(now))
inventory_contents = yaml.dump( self.build_yml_inventory(vms),default_flow_style=False) inventory_contents = yaml.dump(self.build_yml_inventory(vms), default_flow_style=False)
private_key = self.get_private_key(vms) private_key = self.get_private_key(vms)
key_id = self.semaphore_helper.create_ssh_key(application.name, project_id, private_key) key_id = self.semaphore_helper.create_ssh_key(application.name, project_id, private_key)
inventory_id = self.semaphore_helper.create_inventory(application.name, project_id, key_id,inventory_contents) inventory_id = self.semaphore_helper.create_inventory(application.name, project_id, key_id,
inventory_contents)
if 'RUNNING' == desired_state: if 'RUNNING' == desired_state:
standard = interfaces['Standard'] interface = interfaces[interface_type]
create = standard['create'] create = interface['create']
inputs = create['inputs'] inputs = create['inputs']
git_url = inputs['repository'] git_url = inputs['repository']
playbook_names = inputs['resources'] playbook_names = inputs['resources']
for playbook_name in playbook_names: for playbook_name in playbook_names:
task_id = self.run_task(name, project_id, key_id, git_url, inventory_id, playbook_name) environment_id = None
if env_vars:
environment_id = self.semaphore_helper.create_environment(project_id, name, env_vars)
task_id = self.run_task(name, project_id, key_id, git_url, inventory_id, playbook_name,
environment_id=environment_id)
if self.semaphore_helper.get_task(project_id, task_id).status != 'success': if self.semaphore_helper.get_task(project_id, task_id).status != 'success':
break break
# logger.info('playbook: ' + playbook_name + ' task_id: ' + str(task_id))
tasks_outputs[task_id] = self.semaphore_helper.get_task_outputs(project_id, task_id)
if self.semaphore_helper.get_task(project_id,task_id).status == 'success': if 'configure' in interface and self.semaphore_helper.get_task(project_id, task_id).status == 'success':
configure = standard['configure'] configure = interface['configure']
inputs = configure['inputs'] inputs = configure['inputs']
git_url = inputs['repository'] git_url = inputs['repository']
playbook_names = inputs['resources'] playbook_names = inputs['resources']
for playbook_name in playbook_names: for playbook_name in playbook_names:
task_id = self.run_task(name, project_id, key_id, git_url, inventory_id, playbook_name) environment_id = None
if env_vars:
environment_id = self.semaphore_helper.create_environment(project_id, name, env_vars)
task_id = self.run_task(name, project_id, key_id, git_url, inventory_id, playbook_name,
environment_id=environment_id)
if self.semaphore_helper.get_task(project_id, task_id).status != 'success': if self.semaphore_helper.get_task(project_id, task_id).status != 'success':
break break
# logger.info('playbook: ' + playbook_name + ' task_id: ' + str(task_id))
tasks_outputs[task_id] = self.semaphore_helper.get_task_outputs(project_id, task_id)
return tasks_outputs
def build_yml_inventory(self, vms): def build_yml_inventory(self, vms):
# loader = DataLoader() # loader = DataLoader()
...@@ -75,7 +88,7 @@ class AnsibleService: ...@@ -75,7 +88,7 @@ class AnsibleService:
# variable_manager = VariableManager() # variable_manager = VariableManager()
inventory = {} inventory = {}
all = {} all = {}
vars = {'ansible_ssh_common_args':'-o StrictHostKeyChecking=no'} vars = {'ansible_ssh_common_args': '-o StrictHostKeyChecking=no'}
vars['ansible_ssh_user'] = vms[0].node_template.properties['user_name'] vars['ansible_ssh_user'] = vms[0].node_template.properties['user_name']
children = {} children = {}
for vm in vms: for vm in vms:
...@@ -100,24 +113,21 @@ class AnsibleService: ...@@ -100,24 +113,21 @@ class AnsibleService:
private_key = vms[0].node_template.attributes['user_key_pair']['keys']['private_key'] private_key = vms[0].node_template.attributes['user_key_pair']['keys']['private_key']
return base64.b64decode(private_key).decode('utf-8').replace(r'\n', '\n') return base64.b64decode(private_key).decode('utf-8').replace(r'\n', '\n')
def run_task(self, name, project_id, key_id, git_url, inventory_id, playbook_name): def run_task(self, name, project_id, key_id, git_url, inventory_id, playbook_name, environment_id=None):
logger.info('project_id: '+str(project_id)+ ' task name: ' + str(name)+ ' git url: '+git_url+' playbook: '+playbook_name) logger.info('project_id: ' + str(project_id) + ' task name: ' + str(
name) + ' git url: ' + git_url + ' playbook: ' + playbook_name)
self.repository_id = self.semaphore_helper.create_repository(name, project_id, key_id, git_url) self.repository_id = self.semaphore_helper.create_repository(name, project_id, key_id, git_url)
template_id = self.semaphore_helper.create_template(project_id, key_id, inventory_id, self.repository_id, template_id = self.semaphore_helper.create_template(project_id, key_id, inventory_id, self.repository_id,
playbook_name) playbook_name)
task_id = self.semaphore_helper.execute_task(project_id, template_id, playbook_name) task_id = self.semaphore_helper.execute_task(project_id, template_id, playbook_name,
environment_id=environment_id)
task = self.semaphore_helper.get_task(project_id, task_id) task = self.semaphore_helper.get_task(project_id, task_id)
last_output = '' last_status = ''
while task.status == 'waiting' or task.status == 'running': while task.status == 'waiting' or task.status == 'running':
task = self.semaphore_helper.get_task(project_id, task_id) task = self.semaphore_helper.get_task(project_id, task_id)
logger.info('task name: '+name+ ' task status: ' + str(task.status)) this_status = task.status
task_outputs = self.semaphore_helper.get_task_outputs(project_id, task_id) if last_status != this_status:
this_output = task_outputs[len(task_outputs)-1].output.replace(r'\n', '\n').replace(r'\r', '\r') logger.info('task name: ' + name + ', task status: ' + str(task.status))
if last_output != this_output: last_status = this_status
logger.info('task output: ' + str(this_output))
last_output = this_output
# logger.info('task output: ' + str(latask name:st_output))
sleep(3) sleep(3)
return task_id return task_id
deployer/service/deploy_service.py
View file @ 21ce2a62
from service import tosca_helper, ansible_service import json
import logging
from service import tosca_helper
from service.ansible_service import AnsibleService from service.ansible_service import AnsibleService
logger = logging.getLogger(__name__)
if not getattr(logger, 'handler_set', None):
logger.setLevel(logging.INFO)
h = logging.StreamHandler()
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
h.setFormatter(formatter)
logger.addHandler(h)
logger.handler_set = True
class DeployService: class DeployService:
def __init__(self, polemarch_base_url=None,polemarch_username=None,polemarch_password=None, def __init__(self, polemarch_base_url=None,polemarch_username=None,polemarch_password=None,
semaphore_base_url=None,semaphore_username=None,semaphore_password=None): semaphore_base_url=None,semaphore_username=None,semaphore_password=None, vms=None):
# self.polemarch_base_url = polemarch_base_url # self.polemarch_base_url = polemarch_base_url
# self.polemarch_username=polemarch_username # self.polemarch_username=polemarch_username
# self.polemarch_password = polemarch_password # self.polemarch_password = polemarch_password
self.semaphore_base_url = semaphore_base_url self.semaphore_base_url = semaphore_base_url
self.semaphore_username = semaphore_username self.semaphore_username = semaphore_username
self.semaphore_password = semaphore_password self.semaphore_password = semaphore_password
self.vms = vms
for vm in vms:
if vm.node_template.attributes['role'] == 'master':
self.master_ip = vm.node_template.attributes['public_ip']
break
def deploy(self,nodes_pair): def deploy(self,nodes_pair):
target = nodes_pair[0] target = nodes_pair[0]
source = nodes_pair[1] source = nodes_pair[1]
interface_types = tosca_helper.get_interface_types(source) interface_types = tosca_helper.get_interface_types(source)
if interface_types: if interface_types:
ansible_service = AnsibleService(self.semaphore_base_url, self.semaphore_username, self.semaphore_password)
env_vars = self.get_env_vars(nodes_pair)
if 'Standard' in interface_types: if 'Standard' in interface_types:
ansible_service = AnsibleService(self.semaphore_base_url,self.semaphore_username,self.semaphore_password) task_outputs = ansible_service.execute(nodes_pair, 'Standard', self.vms, env_vars=env_vars)
ansible_service.execute(nodes_pair) source = self.set_attributes(task_outputs,source)
else: if 'Kubernetes' in interface_types:
print(interface_types) task_outputs = ansible_service.execute(nodes_pair, 'Kubernetes', self.vms, env_vars=env_vars)
source = self.set_attributes(task_outputs,source)
return source
def get_env_vars(self, nodes_pair):
target = nodes_pair[0]
source = nodes_pair[1]
env_vars = {'K8s_NAMESPACE': 'default'}
if source.node_template.type == 'tosca.nodes.QC.Container.Application.Docker':
env_vars['DOCKER_IMAGE'] = source.node_template.artifacts['image']['file']
env_vars['DOCKER_SERVICE_NAME'] = source.name
env_vars['CONTAINER_PORT'] = source.node_template.properties['ports'][0].split(':')[1]
return env_vars
def set_attributes(self, task_outputs,source):
# target = nodes_pair[0]
# source = nodes_pair[1]
if source.node_template.type == 'tosca.nodes.QC.docker.Orchestrator.Kubernetes':
source = self.set_kubernetes_attributes(source=source,task_outputs=task_outputs)
if source.node_template.type == 'tosca.nodes.QC.Container.Application.Docker':
source = self.set_docker_attributes(source=source, task_outputs=task_outputs)
# lst = list(nodes_pair)
# lst[1] = source
# nodes_pair = tuple(lst)
return source
def parse_ansible_var(self,var_name, output_array):
index = 0
start_index = -1
end_index = -1
for out in output_array:
index += 1
if 'TASK' in out.output or 'PLAY RECAP' in out.output:
if start_index > -1:
end_index = index - 1
break
if start_index <=-1 and '"' + var_name + '":' in out.output:
start_index = index - 1
if start_index <= -1:
return None
ansible_var = output_array[start_index:end_index]
json_ansible_var = '{'
for item in ansible_var:
json_ansible_var = json_ansible_var + item.output
logger.info('found '+var_name +': '+ str(json_ansible_var))
return json.loads(json_ansible_var)
def get_dashboard_token(self, k8s_dashboard_token):
k8s_dashboard_token = k8s_dashboard_token['k8s_dashboard_token']
# if 'resources' in k8s_secrets:
# return self.get_secret_from_k8s_info(k8s_secrets)
if 'stdout' in k8s_dashboard_token:
return self.get_secret_from_stdout(k8s_dashboard_token)
def get_service_port(self, k8s_services, service_name,port_type):
resources = k8s_services['k8s_services']['resources']
for resource in resources:
name = resource['metadata']['name']
if name == service_name:
ports = resource['spec']['ports']
for port in ports:
if port_type in port:
return port[port_type]
return None
def get_secret_from_k8s_info(self, k8s_secrets):
resources = k8s_secrets['resources']
for resource in resources:
metadata = resource['metadata']
if 'admin-user-token' in metadata['name']:
dashboard_token = resource['data']['token']
logger.info('found dashboard_token: ' + str(dashboard_token))
return resource['data']['token']
return None return None
def get_secret_from_stdout(self, k8s_dashboard_token):
return k8s_dashboard_token['stdout'].replace('token: ', '')
def set_kubernetes_attributes(self, source=None,task_outputs=None):
attributes = source.node_template.attributes
if 'tokens' not in attributes:
tokens = []
attributes['tokens'] = tokens
else:
tokens = attributes['tokens']
if 'dashboard_url' not in source.node_template.attributes:
dashboard_url = ''
attributes['dashboard_url'] = dashboard_url
else:
dashboard_url = attributes['dashboard_url']
k8s_dashboard_token = None
k8s_services = None
for task_output_key in task_outputs:
task_output = task_outputs[task_output_key]
if not k8s_dashboard_token:
k8s_dashboard_token = self.parse_ansible_var('k8s_dashboard_token', task_output)
if not k8s_services:
k8s_services = self.parse_ansible_var('k8s_services', task_output)
if k8s_services and k8s_dashboard_token:
credential = {'token_type': 'k8s_dashboard_token'}
credential['token'] = self.get_dashboard_token(k8s_dashboard_token)
tokens.append(credential)
service_port = self.get_service_port(k8s_services, 'kubernetes-dashboard', 'nodePort')
dashboard_url = 'https://' + self.master_ip + ':' + str(service_port)
attributes['dashboard_url'] = dashboard_url
logger.info('source.node_template.attributes: ' + str(attributes))
return source
def set_docker_attributes(self, source, task_outputs):
attributes = source.node_template.attributes
if 'service_url' not in source.node_template.attributes:
service_url = ''
attributes['service_url'] = service_url
for task_output_key in task_outputs:
task_output = task_outputs[task_output_key]
k8s_services = self.parse_ansible_var('k8s_services', task_output)
service_port = self.get_service_port(k8s_services, source.name, 'nodePort')
if service_port:
service_url = 'http://' + self.master_ip + ':' + str(service_port)
attributes['service_url'] = service_url
logger.info('source.node_template.attributes: ' + str(attributes))
return source
deployer/service/tosca_helper.py
View file @ 21ce2a62
import copy
import os import os
import sys import sys
import urllib.request import urllib.request
from sure_tosca_client import Configuration, ApiClient from sure_tosca_client import Configuration, ApiClient, NodeTemplate
from sure_tosca_client.api import default_api from sure_tosca_client.api import default_api
...@@ -28,7 +29,6 @@ class ToscaHelper: ...@@ -28,7 +29,6 @@ class ToscaHelper:
return api return api
def get_interface_types(target): def get_interface_types(target):
print(target.node_template.interfaces)
interface_types = [] interface_types = []
for interface in target.node_template.interfaces: for interface in target.node_template.interfaces:
interface_types.append(interface) interface_types.append(interface)
...@@ -49,7 +49,7 @@ class ToscaHelper: ...@@ -49,7 +49,7 @@ class ToscaHelper:
# So we explicitly get the VMs # So we explicitly get the VMs
# I don't like this solution but I can't think of something better. # I don't like this solution but I can't think of something better.
if related_node.node_template.type == 'tosca.nodes.QC.VM.topology': if related_node.node_template.type == 'tosca.nodes.QC.VM.topology':
vms = self.tosca_client.get_node_templates(self.doc_id,type_name='tosca.nodes.QC.VM.Compute') vms = self.get_vms()
related_node = vms related_node = vms
pair = (related_node, node) pair = (related_node, node)
nodes_pairs.append(pair) nodes_pairs.append(pair)
...@@ -70,6 +70,16 @@ class ToscaHelper: ...@@ -70,6 +70,16 @@ class ToscaHelper:
return True return True
def get_vms(self):
return self.tosca_client.get_node_templates(self.doc_id, type_name='tosca.nodes.QC.VM.Compute')
def set_node(self, updated_node, tosca_template_dict):
node_templates = tosca_template_dict['topology_template']['node_templates']
for node_name in node_templates:
if node_name == updated_node.name:
node_templates[node_name] = updated_node.node_template.to_dict()
return tosca_template_dict
def get_interface_types(node): def get_interface_types(node):
interface_type_names = [] interface_type_names = []
......
deployer/test/test_deployer.py
View file @ 21ce2a62
...@@ -39,7 +39,6 @@ class TestDeployer(unittest.TestCase): ...@@ -39,7 +39,6 @@ class TestDeployer(unittest.TestCase):
sure_tosca_base_url = 'http://127.0.0.1:8081/tosca-sure/1.0.0' sure_tosca_base_url = 'http://127.0.0.1:8081/tosca-sure/1.0.0'
polemarch_base_url='http://127.0.0.1:30001/api/v2'
semaphore_base_url = 'http://127.0.0.1:3000/api' semaphore_base_url = 'http://127.0.0.1:3000/api'
tosca_service_is_up = ToscaHelper.service_is_up(sure_tosca_base_url) tosca_service_is_up = ToscaHelper.service_is_up(sure_tosca_base_url)
semaphore_is_up = ToscaHelper.service_is_up(semaphore_base_url) semaphore_is_up = ToscaHelper.service_is_up(semaphore_base_url)
...@@ -53,8 +52,9 @@ class TestDeployer(unittest.TestCase): ...@@ -53,8 +52,9 @@ class TestDeployer(unittest.TestCase):
self.assertIsNotNone(nodes_pairs) self.assertIsNotNone(nodes_pairs)
username = 'admin' username = 'admin'
deployService = DeployService(polemarch_base_url=polemarch_base_url,polemarch_username=username,polemarch_password='admin', deployService = DeployService(polemarch_username=username,polemarch_password='admin',
semaphore_base_url=semaphore_base_url,semaphore_username=username,semaphore_password='password') semaphore_base_url=semaphore_base_url,semaphore_username=username,
semaphore_password='password')
for node_pair in nodes_pairs: for node_pair in nodes_pairs:
deployService.deploy(node_pair) deployService.deploy(node_pair)
......
docker-compose/docker-compose.yml
View file @ 21ce2a62
version: '3' version: '3'
services: services:
#nginx: nginx:
#image: nginx image: nginx
#volumes: volumes:
#- ./nginx.conf:/etc/nginx/nginx.conf - ./nginx.conf:/etc/nginx/nginx.conf
#- ./cert.pem:/etc/nginx/cert.pem - ./cert.pem:/etc/nginx/cert.pem
#- ./privkey.pem:/etc/nginx/privkey.pem - ./privkey.pem:/etc/nginx/privkey.pem
##- ./www:/data/www #- ./www:/data/www
#ports: ports:
#- "80:80" - "80:80"
#- "443:443" - "443:443"
rabbit: rabbit:
image: rabbitmq:3.8-management image: rabbitmq:3.8-management
...@@ -55,21 +55,22 @@ services: ...@@ -55,21 +55,22 @@ services:
- "27017:27017" - "27017:27017"
#manager: manager:
#depends_on: depends_on:
#- rabbit - rabbit
#- mongo - mongo
#- sure-tosca - sure-tosca
#image: qcdis/manager:3.0.0 image: qcdis/manager
#environment: environment:
#RABBITMQ_HOST: rabbit RABBITMQ_HOST: rabbit
#MONGO_HOST: mongo MONGO_HOST: mongo
#SURE_TOSCA_BASE_PATH: http://sure-tosca:8081/tosca-sure/1.0.0 SURE_TOSCA_BASE_PATH: http://sure-tosca:8081/tosca-sure/1.0.0
#ports: CREDENTIAL_SECRET: 123
#- "8080:8080" ports:
- "8080:8080"
sure-tosca: sure-tosca:
image: qcdis/sure-tosca:3.0.0 image: qcdis/sure-tosca
ports: ports:
- "8081:8081" - "8081:8081"
...@@ -77,27 +78,32 @@ services: ...@@ -77,27 +78,32 @@ services:
depends_on: depends_on:
- rabbit - rabbit
- sure-tosca - sure-tosca
image: qcdis/planner:3.0.0 image: qcdis/planner
environment:
RABBITMQ_HOST: rabbit
provisioner:
depends_on:
- rabbit
- sure-tosca
image: qcdis/provisioner
environment: environment:
RABBITMQ_HOST: rabbit RABBITMQ_HOST: rabbit
SURE_TOSCA_BASE_PATH: http://sure-tosca:8081/tosca-sure/1.0.0
CLOUD_STORM_SECRET: 456
CREDENTIAL_SECRET: 123
deployer:
depends_on:
- rabbit
- sure-tosca
image: qcdis/deployer
environment:
RABBITMQ_HOST: rabbit
SURE_TOSCA_BASE_PATH: http://sure-tosca:8081/tosca-sure/1.0.0
SEMAPHORE_BASE_PATH: http://semaphore:3000/api
#provisioner:
#depends_on:
#- rabbit
#- sure-tosca
#image: qcdis/provisioner:3.0.0
#environment:
#RABBITMQ_HOST: rabbit
#SURE_TOSCA_BASE_PATH: http://sure-tosca:8081/tosca-sure/1.0.0
#deployer:
#depends_on:
#- rabbit
#- sure-tosca
#image: qcdis/deployer:3.0.0
#environment:
#RABBITMQ_HOST: rabbit
#SURE_TOSCA_BASE_PATH: http://sure-tosca:8081/tosca-sure/1.0.0
docker-compose/nginx.conf
View file @ 21ce2a62
worker_processes auto; worker_processes auto;
events { events {
worker_connections 1024; worker_connections 1024;
use epoll; use epoll;
...@@ -7,19 +8,17 @@ events { ...@@ -7,19 +8,17 @@ events {
} }
http { http {
proxy_connect_timeout 9200;
proxy_send_timeout 9200;
proxy_read_timeout 9200;
send_timeout 9200;
tcp_nodelay on; tcp_nodelay on;
# this is necessary for us to be able to disable request buffering in all cases # this is necessary for us to be able to disable request buffering in all cases
proxy_http_version 1.1; proxy_http_version 1.1;
# upstream semaphore {
# server semaphore:3000;
# }
# upstream manager {
# server manager:8080;
# }
server { server {
listen [::]:80 default_server; listen [::]:80 default_server;
server_name _; server_name _;
...@@ -61,16 +60,9 @@ http { ...@@ -61,16 +60,9 @@ http {
proxy_request_buffering off; proxy_request_buffering off;
} }
# location /api/ws {
# proxy_pass http://semaphore:3000/api/ws;
# proxy_http_version 1.1;
# proxy_set_header Upgrade $http_upgrade;
# proxy_set_header Connection "upgrade";
# proxy_set_header Origin "";
# }
location /manger/ { location /manager {
proxy_pass http://manager:8080/manger/; proxy_pass http://manager:8080/manager;
proxy_set_header Host $http_host; proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
...@@ -81,8 +73,8 @@ http { ...@@ -81,8 +73,8 @@ http {
proxy_request_buffering off; proxy_request_buffering off;
} }
location /manger/swagger-ui.html { location /swagger-ui.html {
proxy_pass http://manager:8080/swagger-ui.html/; proxy_pass http://manager:8080/swagger-ui.html;
proxy_set_header Host $http_host; proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
...@@ -93,8 +85,8 @@ http { ...@@ -93,8 +85,8 @@ http {
proxy_request_buffering off; proxy_request_buffering off;
} }
location /tosca-sure/1.0.0/ { location /tosca-sure/1.0.0 {
proxy_pass http://sure-tosca:8081/tosca-sure/1.0.0/; proxy_pass http://sure-tosca:8081/tosca-sure/1.0.0;
proxy_set_header Host $http_host; proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
......
example_messages/message_provision_request.json
View file @ 21ce2a62
{ {
"owner": "user", "owner": "user",
"creationDate": 1585660972026, "creationDate": 1591357712632,
"toscaTemplate": { "toscaTemplate": {
"tosca_definitions_version": "tosca_simple_yaml_1_0", "tosca_definitions_version": "tosca_simple_yaml_1_0",
"tosca_default_namespace": null, "tosca_default_namespace": null,
...@@ -11,7 +11,7 @@ ...@@ -11,7 +11,7 @@
"node_templates": { "node_templates": {
"compute": { "compute": {
"properties": { "properties": {
"disk_size": "10000 MB", "disk_size": "20000 MB",
"mem_size": "1000 MB", "mem_size": "1000 MB",
"num_cores": 1, "num_cores": 1,
"os": "Ubuntu 18.04", "os": "Ubuntu 18.04",
...@@ -26,7 +26,7 @@ ...@@ -26,7 +26,7 @@
}, },
"compute_1": { "compute_1": {
"properties": { "properties": {
"disk_size": "10000 MB", "disk_size": "20000 MB",
"mem_size": "1000 MB", "mem_size": "1000 MB",
"num_cores": 1, "num_cores": 1,
"os": "Ubuntu 18.04", "os": "Ubuntu 18.04",
...@@ -39,7 +39,11 @@ ...@@ -39,7 +39,11 @@
}, },
"type": "tosca.nodes.QC.VM.Compute" "type": "tosca.nodes.QC.VM.Compute"
}, },
"kubernetes": { "glusterfs": {
"properties": {
"min_masters_num": 1,
"min_workers_num": 1
},
"requirements": [ "requirements": [
{ {
"host": { "host": {
...@@ -53,25 +57,60 @@ ...@@ -53,25 +57,60 @@
"Standard": { "Standard": {
"configure": { "configure": {
"inputs": { "inputs": {
"playbook": "k8s_dashboard.yaml", "repository": "https://github.com/QCDIS/mysome_glusterfs.git",
"repository": "https://github.com/skoulouzis/playbooks.git" "resources": [
"002.setup_glusterfs_infra.yml",
"003.setup_glusterfs_cluster.yml",
"004.mount_glusterfs.yml"
]
} }
}, },
"create": { "create": {
"inputs": { "inputs": {
"playbook": "install_k8s.yaml", "repository": "https://github.com/QCDIS/playbooks.git",
"repository": "https://github.com/skoulouzis/playbooks.git" "resources": [
"ec2/ec2_mount.yaml",
"install_glusterfs_prerequisites.yaml"
]
}
}
} }
}, },
"delete": { "type": "tosca.nodes.QC.Application.GlusterFS"
},
"tic": {
"requirements": [
{
"storage": {
"capability": "tosca.capabilities.QC.GlusterFS",
"node": "glusterfs",
"relationship": "tosca.relationships.DependsOn"
}
}
],
"interfaces": {
"Standard": {
"create": {
"inputs": { "inputs": {
"playbook": "k8s_delete.yaml", "repository": "https://github.com/bityoga/fabric_as_code.git",
"repository": "https://github.com/skoulouzis/playbooks.git" "resources": [
"011.initialize_hosts.yml",
"012.prepare_docker_images.yml",
"013.mount_fs.yml",
"014.spawn_swarm.yml",
"015.deploy_swarm_visualizer.yml",
"016.deploy_portainer.yml",
"100.deploy_ca.yml",
"101.deploy_orderer.yml",
"102.deploy_peers.yml",
"103.deploy_cli.yml",
"104.deploy_hlf_explorer.yml"
]
} }
} }
} }
}, },
"type": "tosca.nodes.QC.docker.Orchestrator.Kubernetes" "type": "tosca.nodes.QC.Application.TIC"
}, },
"topology": { "topology": {
"properties": { "properties": {
...@@ -133,9 +172,9 @@ ...@@ -133,9 +172,9 @@
"credential": { "credential": {
"cloud_provider_name": "EC2", "cloud_provider_name": "EC2",
"keys": { "keys": {
"aws_access_key_id": "XXXXXXXXXXXX" "aws_access_key_id": "BGCAc9htRRjcZx5uZ6aWnRo67tNAgk7M26DwuN3cZvypnCt6V56U9RbEuRpmXbER"
}, },
"token": "XXXXXXXXXXXXX", "token": "BGCAc9htRRjcZx5uZ6aWnRo67tNAgk7M26DwuN3cZvypnCt6V56U9RbEuRpmXbER",
"token_type": "access_key" "token_type": "access_key"
}, },
"desired_state": "RUNNING" "desired_state": "RUNNING"
...@@ -146,30 +185,6 @@ ...@@ -146,30 +185,6 @@
"type": "string" "type": "string"
} }
} }
},
"ws-pema": {
"properties": {
"ports": [
"30001:8080"
]
},
"requirements": [
{
"host": {
"capability": "tosca.capabilities.QC.docker.Orchestrator",
"node": "kubernetes",
"relationship": "tosca.relationships.HostedOn"
}
}
],
"type": "tosca.nodes.QC.Container.Application.Docker",
"artifacts": {
"image": {
"file": "alogo53/ws-pema-lifewatch",
"repository": "docker_hub",
"type": "tosca.artifacts.Deployment.Image.Container.Docker"
}
}
} }
}, },
"relationship_templates": null, "relationship_templates": null,
...@@ -183,19 +198,19 @@ ...@@ -183,19 +198,19 @@
"description": "TOSCA example", "description": "TOSCA example",
"imports": [ "imports": [
{ {
"nodes": "https://raw.githubusercontent.com/skoulouzis/DRIP/develop/TOSCA/types/nodes.yaml" "nodes": "https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/types/nodes.yaml"
}, },
{ {
"data": "https://raw.githubusercontent.com/skoulouzis/CONF/develop/TOSCA/types/data.yml" "data": "https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/types/data.yml"
}, },
{ {
"capabilities": "https://raw.githubusercontent.com/skoulouzis/DRIP/develop/TOSCA/types/capabilities.yaml" "capabilities": "https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/types/capabilities.yaml"
}, },
{ {
"policies": "https://raw.githubusercontent.com/skoulouzis/DRIP/develop/TOSCA/types/policies.yaml" "policies": "https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/types/policies.yaml"
}, },
{ {
"interfaces": "https://raw.githubusercontent.com/skoulouzis/DRIP/develop/TOSCA/types/interfaces.yml" "interfaces": "https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/types/interfaces.yml"
} }
], ],
"dsl_definitions": null, "dsl_definitions": null,
......
k8s/CONF/deployer-deployment.yaml
View file @ 21ce2a62
...@@ -33,7 +33,9 @@ spec: ...@@ -33,7 +33,9 @@ spec:
value: rabbit value: rabbit
- name: SURE_TOSCA_BASE_PATH - name: SURE_TOSCA_BASE_PATH
value: http://sure-tosca:8081/tosca-sure/1.0.0 value: http://sure-tosca:8081/tosca-sure/1.0.0
image: qcdis/deployer:3.0.0 - name: SEMAPHORE_BASE_PATH
value: http://semaphore:3000/api
image: qcdis/deployer
name: deployer name: deployer
imagePullPolicy: Always imagePullPolicy: Always
resources: {} resources: {}
......
k8s/CONF/manager-deployment.yaml
View file @ 21ce2a62
...@@ -35,6 +35,8 @@ spec: ...@@ -35,6 +35,8 @@ spec:
value: rabbit value: rabbit
- name: SURE_TOSCA_BASE_PATH - name: SURE_TOSCA_BASE_PATH
value: http://sure-tosca:8081/tosca-sure/1.0.0 value: http://sure-tosca:8081/tosca-sure/1.0.0
- name: CREDENTIAL_SECRET
value: top_secret
image: qcdis/manager:3.0.0 image: qcdis/manager:3.0.0
name: manager name: manager
imagePullPolicy: Always imagePullPolicy: Always
......
k8s/CONF/manager-deployment.yaml.template 0 → 100644
View file @ 21ce2a62
apiVersion: apps/v1
kind: Deployment
metadata:
namespace: conf
annotations:
kompose.cmd: kompose convert
kompose.version: 1.16.0 (0c01309)
creationTimestamp: null
labels:
io.kompose.service: manager
name: manager
spec:
selector:
matchLabels:
io.kompose.service: manager
replicas: 1
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 50%
maxUnavailable: 50%
minReadySeconds: 10
revisionHistoryLimit: 3
template:
metadata:
creationTimestamp: null
labels:
io.kompose.service: manager
spec:
containers:
- env:
- name: MONGO_HOST
value: mongo
- name: RABBITMQ_HOST
value: rabbit
- name: SURE_TOSCA_BASE_PATH
value: http://sure-tosca:8081/tosca-sure/1.0.0
- name: CREDENTIAL_SECRET
value: top_secret
image: qcdis/manager:3.0.0
name: manager
imagePullPolicy: Always
ports:
- containerPort: 8080
resources: {}
restartPolicy: Always
status: {}
k8s/CONF/nginx-service.yaml
View file @ 21ce2a62
...@@ -8,10 +8,6 @@ metadata: ...@@ -8,10 +8,6 @@ metadata:
spec: spec:
type: NodePort type: NodePort
ports: ports:
- port: 80
nodePort: 30000
protocol: TCP
name: http
- port: 443 - port: 443
nodePort: 30001 nodePort: 30001
protocol: TCP protocol: TCP
......
k8s/CONF/planner-deployment.yaml
View file @ 21ce2a62
...@@ -31,7 +31,7 @@ spec: ...@@ -31,7 +31,7 @@ spec:
- env: - env:
- name: RABBITMQ_HOST - name: RABBITMQ_HOST
value: rabbit value: rabbit
image: qcdis/planner:3.0.0 image: qcdis/planner
name: planner name: planner
imagePullPolicy: Always imagePullPolicy: Always
resources: {} resources: {}
......
k8s/CONF/provisioner-deployment.yaml.template 0 → 100644
View file @ 21ce2a62
apiVersion: apps/v1
kind: Deployment
metadata:
namespace: conf
annotations:
kompose.cmd: kompose convert
kompose.version: 1.16.0 (0c01309)
creationTimestamp: null
labels:
io.kompose.service: provisioner
name: provisioner
spec:
selector:
matchLabels:
io.kompose.service: provisioner
replicas: 1
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 50%
maxUnavailable: 50%
minReadySeconds: 10
revisionHistoryLimit: 3
template:
metadata:
creationTimestamp: null
labels:
io.kompose.service: provisioner
spec:
containers:
- env:
- name: RABBITMQ_HOST
value: rabbit
- name: SURE_TOSCA_BASE_PATH
value: http://sure-tosca:8081/tosca-sure/1.0.0
image: qcdis/provisioner:3.0.0
name: provisioner
imagePullPolicy: Always
resources: {}
restartPolicy: Always
status: {}
k8s/CONF/sure-tosca-deployment.yaml
View file @ 21ce2a62
...@@ -28,7 +28,7 @@ spec: ...@@ -28,7 +28,7 @@ spec:
io.kompose.service: sure-tosca io.kompose.service: sure-tosca
spec: spec:
containers: containers:
- image: qcdis/sure-tosca:3.0.0 - image: qcdis/sure-tosca
name: sure-tosca name: sure-tosca
imagePullPolicy: Always imagePullPolicy: Always
ports: ports:
......
manager/Dockerfile
View file @ 21ce2a62
...@@ -5,6 +5,7 @@ CMD jar -xf manager-3.0.0.jar BOOT-INF/classes/application.properties && \ ...@@ -5,6 +5,7 @@ CMD jar -xf manager-3.0.0.jar BOOT-INF/classes/application.properties && \
sed -ie "s#^message.broker.host=.*#message.broker.host=$RABBITMQ_HOST#" BOOT-INF/classes/application.properties && \ sed -ie "s#^message.broker.host=.*#message.broker.host=$RABBITMQ_HOST#" BOOT-INF/classes/application.properties && \
sed -ie "s#^db.host=.*#db.host=$MONGO_HOST#" BOOT-INF/classes/application.properties && \ sed -ie "s#^db.host=.*#db.host=$MONGO_HOST#" BOOT-INF/classes/application.properties && \
sed -ie "s#^sure-tosca.base.path=.*#sure-tosca.base.path=$SURE_TOSCA_BASE_PATH#" BOOT-INF/classes/application.properties && \ sed -ie "s#^sure-tosca.base.path=.*#sure-tosca.base.path=$SURE_TOSCA_BASE_PATH#" BOOT-INF/classes/application.properties && \
sed -ie "s#^credential.secret=.*#credential.secret=$CREDENTIAL_SECRET#" BOOT-INF/classes/application.properties && \
cat BOOT-INF/classes/application.properties && \ cat BOOT-INF/classes/application.properties && \
jar -uf manager-3.0.0.jar BOOT-INF/classes/application.properties && \ jar -uf manager-3.0.0.jar BOOT-INF/classes/application.properties && \
java -jar manager-3.0.0.jar java -jar manager-3.0.0.jar
manager/src/main/java/nl/uva/sne/drip/api/CredentialApiController.java
View file @ 21ce2a62
...@@ -3,6 +3,9 @@ package nl.uva.sne.drip.api; ...@@ -3,6 +3,9 @@ package nl.uva.sne.drip.api;
import nl.uva.sne.drip.model.tosca.Credential; import nl.uva.sne.drip.model.tosca.Credential;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import io.swagger.annotations.*; import io.swagger.annotations.*;
import java.io.UnsupportedEncodingException;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.http.HttpStatus; import org.springframework.http.HttpStatus;
...@@ -12,6 +15,10 @@ import org.springframework.web.bind.annotation.RequestBody; ...@@ -12,6 +15,10 @@ import org.springframework.web.bind.annotation.RequestBody;
import javax.validation.Valid; import javax.validation.Valid;
import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletRequest;
import java.util.List; import java.util.List;
import java.util.logging.Level;
import javax.crypto.BadPaddingException;
import javax.crypto.IllegalBlockSizeException;
import javax.crypto.NoSuchPaddingException;
import nl.uva.sne.drip.service.CredentialService; import nl.uva.sne.drip.service.CredentialService;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
...@@ -41,8 +48,12 @@ public class CredentialApiController implements CredentialApi { ...@@ -41,8 +48,12 @@ public class CredentialApiController implements CredentialApi {
@Valid @RequestBody Credential body) { @Valid @RequestBody Credential body) {
String accept = request.getHeader("Accept"); String accept = request.getHeader("Accept");
if (accept != null && accept.contains("application/json")) { if (accept != null && accept.contains("application/json")) {
try {
String id = credentialService.save(body); String id = credentialService.save(body);
return new ResponseEntity<>(id, HttpStatus.OK); return new ResponseEntity<>(id, HttpStatus.OK);
} catch (UnsupportedEncodingException | NoSuchAlgorithmException | NoSuchPaddingException | InvalidKeyException | IllegalBlockSizeException | BadPaddingException ex) {
return new ResponseEntity<>(HttpStatus.INTERNAL_SERVER_ERROR);
}
} else { } else {
return new ResponseEntity<>(HttpStatus.NOT_ACCEPTABLE); return new ResponseEntity<>(HttpStatus.NOT_ACCEPTABLE);
} }
......
manager/src/main/java/nl/uva/sne/drip/api/ToscaTemplateApiController.java
View file @ 21ce2a62
...@@ -54,7 +54,7 @@ public class ToscaTemplateApiController implements ToscaTemplateApi { ...@@ -54,7 +54,7 @@ public class ToscaTemplateApiController implements ToscaTemplateApi {
if (accept != null && accept.contains("text/plain")) { if (accept != null && accept.contains("text/plain")) {
try { try {
dripService.delete(id, nodeName); dripService.delete(id, nodeName);
return new ResponseEntity<>("", HttpStatus.OK); return new ResponseEntity<>(id, HttpStatus.OK);
} catch (IOException | ApiException | TypeExeption | TimeoutException | InterruptedException ex) { } catch (IOException | ApiException | TypeExeption | TimeoutException | InterruptedException ex) {
java.util.logging.Logger.getLogger(ToscaTemplateApiController.class.getName()).log(Level.SEVERE, null, ex); java.util.logging.Logger.getLogger(ToscaTemplateApiController.class.getName()).log(Level.SEVERE, null, ex);
return new ResponseEntity<>(HttpStatus.INTERNAL_SERVER_ERROR); return new ResponseEntity<>(HttpStatus.INTERNAL_SERVER_ERROR);
......
manager/src/main/java/nl/uva/sne/drip/service/CredentialService.java
View file @ 21ce2a62
...@@ -5,13 +5,22 @@ ...@@ -5,13 +5,22 @@
*/ */
package nl.uva.sne.drip.service; package nl.uva.sne.drip.service;
import com.fasterxml.jackson.core.JsonProcessingException; import java.io.UnsupportedEncodingException;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.Map;
import java.util.Set;
import javax.crypto.BadPaddingException;
import javax.crypto.IllegalBlockSizeException;
import javax.crypto.NoSuchPaddingException;
import nl.uva.sne.drip.commons.utils.Converter;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import nl.uva.sne.drip.dao.CredentialDAO; import nl.uva.sne.drip.dao.CredentialDAO;
import nl.uva.sne.drip.model.tosca.Credential; import nl.uva.sne.drip.model.tosca.Credential;
import org.springframework.beans.factory.annotation.Value;
/** /**
* *
...@@ -20,17 +29,21 @@ import nl.uva.sne.drip.model.tosca.Credential; ...@@ -20,17 +29,21 @@ import nl.uva.sne.drip.model.tosca.Credential;
@Service @Service
public class CredentialService { public class CredentialService {
@Value("${credential.secret}")
private String credentialSecret;
@Autowired @Autowired
private CredentialDAO dao; private CredentialDAO dao;
public String save(Credential document) { public String save(Credential document) throws UnsupportedEncodingException, NoSuchAlgorithmException, NoSuchPaddingException, InvalidKeyException, IllegalBlockSizeException, BadPaddingException {
dao.save(document);
dao.save(Converter.encryptCredential(document,credentialSecret));
return document.getId(); return document.getId();
} }
public Credential findByID(String id) throws JsonProcessingException { public Credential findByID(String id) {
Credential credentials = dao.findById(id).get(); Credential credential = dao.findById(id).get();
return credentials; return credential;
} }
public void deleteByID(String id) { public void deleteByID(String id) {
...@@ -54,4 +67,6 @@ public class CredentialService { ...@@ -54,4 +67,6 @@ public class CredentialService {
return dao.findBycloudProviderName(provider); return dao.findBycloudProviderName(provider);
} }
} }
manager/src/main/java/nl/uva/sne/drip/service/DRIPService.java
View file @ 21ce2a62
...@@ -67,8 +67,8 @@ public class DRIPService { ...@@ -67,8 +67,8 @@ public class DRIPService {
message.setToscaTemplate(toscaTemplate); message.setToscaTemplate(toscaTemplate);
caller.setRequestQeueName(requestQeueName); caller.setRequestQeueName(requestQeueName);
Message plannerResponse = caller.call(message); Message response = caller.call(message);
ToscaTemplate updatedToscaTemplate = plannerResponse.getToscaTemplate(); ToscaTemplate updatedToscaTemplate = response.getToscaTemplate();
caller.close(); caller.close();
return toscaTemplateService.save(updatedToscaTemplate); return toscaTemplateService.save(updatedToscaTemplate);
......
manager/src/main/resources/application.properties
View file @ 21ce2a62
...@@ -19,3 +19,5 @@ db.username=drip-user ...@@ -19,3 +19,5 @@ db.username=drip-user
db.password=drip-pass db.password=drip-pass
sure-tosca.base.path=http://localhost:8081/tosca-sure/1.0.0 sure-tosca.base.path=http://localhost:8081/tosca-sure/1.0.0
credential.secret=top_secret
\ No newline at end of file
manager/src/test/java/nl/uva/sne/drip/service/ServiceTests.java
View file @ 21ce2a62
...@@ -28,6 +28,8 @@ import java.io.File; ...@@ -28,6 +28,8 @@ import java.io.File;
import java.io.FileInputStream; import java.io.FileInputStream;
import java.io.FileNotFoundException; import java.io.FileNotFoundException;
import java.io.IOException; import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException; import java.security.NoSuchAlgorithmException;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
...@@ -37,6 +39,9 @@ import java.util.Set; ...@@ -37,6 +39,9 @@ import java.util.Set;
import java.util.concurrent.TimeoutException; import java.util.concurrent.TimeoutException;
import java.util.logging.Level; import java.util.logging.Level;
import java.util.logging.Logger; import java.util.logging.Logger;
import javax.crypto.BadPaddingException;
import javax.crypto.IllegalBlockSizeException;
import javax.crypto.NoSuchPaddingException;
import nl.uva.sne.drip.Swagger2SpringBoot; import nl.uva.sne.drip.Swagger2SpringBoot;
import nl.uva.sne.drip.api.NotFoundException; import nl.uva.sne.drip.api.NotFoundException;
import nl.uva.sne.drip.commons.utils.Constants; import nl.uva.sne.drip.commons.utils.Constants;
...@@ -301,7 +306,7 @@ public class ServiceTests { ...@@ -301,7 +306,7 @@ public class ServiceTests {
* Test of save method, of class CredentialService. * Test of save method, of class CredentialService.
*/ */
@Test @Test
public void testCredentialServiceSave() { public void testCredentialServiceSave() throws UnsupportedEncodingException, NoSuchAlgorithmException, NoSuchPaddingException, InvalidKeyException, IllegalBlockSizeException, BadPaddingException {
Logger.getLogger(ServiceTests.class.getName()).log(Level.INFO, "save"); Logger.getLogger(ServiceTests.class.getName()).log(Level.INFO, "save");
saveCredential(); saveCredential();
} }
...@@ -332,7 +337,7 @@ public class ServiceTests { ...@@ -332,7 +337,7 @@ public class ServiceTests {
assertEquals(keystorFileChecksum, keystorFileCopyChecksum); assertEquals(keystorFileChecksum, keystorFileCopyChecksum);
} }
public String saveCredential() { public String saveCredential() throws UnsupportedEncodingException, NoSuchAlgorithmException, NoSuchPaddingException, InvalidKeyException, IllegalBlockSizeException, BadPaddingException {
Logger.getLogger(ServiceTests.class.getName()).log(Level.INFO, "saveCredential"); Logger.getLogger(ServiceTests.class.getName()).log(Level.INFO, "saveCredential");
Credential document = new Credential(); Credential document = new Credential();
document.setCloudProviderName("exogeni"); document.setCloudProviderName("exogeni");
...@@ -364,7 +369,7 @@ public class ServiceTests { ...@@ -364,7 +369,7 @@ public class ServiceTests {
* @throws com.fasterxml.jackson.core.JsonProcessingException * @throws com.fasterxml.jackson.core.JsonProcessingException
*/ */
@Test @Test
public void testCredentialServiceDeleteByID() throws JsonProcessingException { public void testCredentialServiceDeleteByID() throws JsonProcessingException, UnsupportedEncodingException, NoSuchAlgorithmException, NoSuchPaddingException, InvalidKeyException, IllegalBlockSizeException, BadPaddingException {
Logger.getLogger(ServiceTests.class.getName()).log(Level.INFO, "deleteByID"); Logger.getLogger(ServiceTests.class.getName()).log(Level.INFO, "deleteByID");
String id = saveCredential(); String id = saveCredential();
credentialService.deleteByID(id); credentialService.deleteByID(id);
...@@ -382,7 +387,7 @@ public class ServiceTests { ...@@ -382,7 +387,7 @@ public class ServiceTests {
* Test of getAllIds method, of class CredentialService. * Test of getAllIds method, of class CredentialService.
*/ */
@Test @Test
public void testCredentialServiceGetAllIds() { public void testCredentialServiceGetAllIds() throws UnsupportedEncodingException, NoSuchAlgorithmException, NoSuchPaddingException, InvalidKeyException, IllegalBlockSizeException, BadPaddingException {
Logger.getLogger(ServiceTests.class.getName()).log(Level.INFO, "getAllIds"); Logger.getLogger(ServiceTests.class.getName()).log(Level.INFO, "getAllIds");
testCredentialServiceDeleteAll(); testCredentialServiceDeleteAll();
int numOfINst = 3; int numOfINst = 3;
...@@ -404,7 +409,7 @@ public class ServiceTests { ...@@ -404,7 +409,7 @@ public class ServiceTests {
} }
@Test @Test
public void testSetProvisionerOperation() throws FileNotFoundException, IOException, MissingCredentialsException, ApiException, TypeExeption, JsonProcessingException, TimeoutException, InterruptedException, NotFoundException, MissingVMTopologyException { public void testSetProvisionerOperation() throws FileNotFoundException, IOException, MissingCredentialsException, ApiException, TypeExeption, JsonProcessingException, TimeoutException, InterruptedException, NotFoundException, MissingVMTopologyException, UnsupportedEncodingException, NoSuchAlgorithmException, NoSuchPaddingException, InvalidKeyException, IllegalBlockSizeException, BadPaddingException {
if (ToscaHelper.isServiceUp(sureToscaBasePath) && ToscaHelper.isServiceUp("http://" + messageBrokerHost + ":15672")) { if (ToscaHelper.isServiceUp(sureToscaBasePath) && ToscaHelper.isServiceUp("http://" + messageBrokerHost + ":15672")) {
addRandomCredential("ExoGENI"); addRandomCredential("ExoGENI");
...@@ -447,7 +452,7 @@ public class ServiceTests { ...@@ -447,7 +452,7 @@ public class ServiceTests {
} }
} }
private void addRandomCredential(String providerName) { private void addRandomCredential(String providerName) throws UnsupportedEncodingException, NoSuchAlgorithmException, NoSuchPaddingException, InvalidKeyException, IllegalBlockSizeException, BadPaddingException {
Credential document = new Credential(); Credential document = new Credential();
document.setCloudProviderName(providerName); document.setCloudProviderName(providerName);
Map<String, String> keys = new HashMap<>(); Map<String, String> keys = new HashMap<>();
......
provisioner/Dockerfile
View file @ 21ce2a62
...@@ -6,8 +6,14 @@ CMD jar -xf provisioner-3.0.0-jar-with-dependencies.jar application.properties & ...@@ -6,8 +6,14 @@ CMD jar -xf provisioner-3.0.0-jar-with-dependencies.jar application.properties &
cat application.properties && \ cat application.properties && \
sed -ie "s#^message.broker.host=.*#message.broker.host=$RABBITMQ_HOST#" application.properties && \ sed -ie "s#^message.broker.host=.*#message.broker.host=$RABBITMQ_HOST#" application.properties && \
sed -ie "s#^sure-tosca.base.path=.*#sure-tosca.base.path=$SURE_TOSCA_BASE_PATH#" application.properties && \ sed -ie "s#^sure-tosca.base.path=.*#sure-tosca.base.path=$SURE_TOSCA_BASE_PATH#" application.properties && \
sed -ie "s#^cloud.storm.secret=.*#cloud.storm.secret=$CLOUD_STORM_SECRET#" application.properties && \
sed -ie "s#^credential.secret=.*#credential.secret=$CREDENTIAL_SECRET#" application.properties && \
sed -ie "s#^cloud.storm.db.path=.*#cloud.storm.db.path=/etc/UD#" application.properties && \
echo "" >> application.properties && \
echo "cloud.storm.db.path=/etc/UD" >> application.properties && \ echo "cloud.storm.db.path=/etc/UD" >> application.properties && \
cat application.properties && \ cat application.properties && \
jar -uf provisioner-3.0.0-jar-with-dependencies.jar application.properties && \ jar -uf provisioner-3.0.0-jar-with-dependencies.jar application.properties && \
sleep 5 && \ sleep 5 && \
java -jar provisioner-3.0.0-jar-with-dependencies.jar java -jar provisioner-3.0.0-jar-with-dependencies.jar
provisioner/src/main/java/nl/uva/sne/drip/provisioner/CloudStormService.java
View file @ 21ce2a62
...@@ -17,9 +17,12 @@ import com.jcraft.jsch.KeyPair; ...@@ -17,9 +17,12 @@ import com.jcraft.jsch.KeyPair;
import java.io.File; import java.io.File;
import java.io.FileNotFoundException; import java.io.FileNotFoundException;
import java.io.IOException; import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Paths; import java.nio.file.Paths;
import java.nio.file.attribute.PosixFilePermission; import java.nio.file.attribute.PosixFilePermission;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.HashSet; import java.util.HashSet;
...@@ -29,6 +32,9 @@ import java.util.Properties; ...@@ -29,6 +32,9 @@ import java.util.Properties;
import java.util.Set; import java.util.Set;
import java.util.logging.Level; import java.util.logging.Level;
import java.util.logging.Logger; import java.util.logging.Logger;
import javax.crypto.BadPaddingException;
import javax.crypto.IllegalBlockSizeException;
import javax.crypto.NoSuchPaddingException;
import nl.uva.sne.drip.commons.utils.Constants; import nl.uva.sne.drip.commons.utils.Constants;
import static nl.uva.sne.drip.commons.utils.Constants.*; import static nl.uva.sne.drip.commons.utils.Constants.*;
import nl.uva.sne.drip.commons.utils.Converter; import nl.uva.sne.drip.commons.utils.Converter;
...@@ -62,6 +68,9 @@ import topology.analysis.TopologyAnalysisMain; ...@@ -62,6 +68,9 @@ import topology.analysis.TopologyAnalysisMain;
*/ */
class CloudStormService { class CloudStormService {
protected String secret;
private String credentialSecret;
/** /**
* @return the helper * @return the helper
*/ */
...@@ -105,6 +114,15 @@ class CloudStormService { ...@@ -105,6 +114,15 @@ class CloudStormService {
if (sureToscaBasePath == null) { if (sureToscaBasePath == null) {
throw new NullPointerException("sureToscaBasePath cannot be null"); throw new NullPointerException("sureToscaBasePath cannot be null");
} }
secret = properties.getProperty("cloud.storm.secret");
if (secret == null) {
throw new NullPointerException("secret cannot be null");
}
credentialSecret = properties.getProperty("credential.secret");
if (credentialSecret == null) {
throw new NullPointerException("secret cannot be null");
}
Logger.getLogger(CloudStormService.class.getName()).log(Level.FINE, "sureToscaBasePath: {0}", sureToscaBasePath); Logger.getLogger(CloudStormService.class.getName()).log(Level.FINE, "sureToscaBasePath: {0}", sureToscaBasePath);
this.helper = new ToscaHelper(sureToscaBasePath); this.helper = new ToscaHelper(sureToscaBasePath);
this.helper.uploadToscaTemplate(toscaTemplate); this.helper.uploadToscaTemplate(toscaTemplate);
...@@ -123,9 +141,10 @@ class CloudStormService { ...@@ -123,9 +141,10 @@ class CloudStormService {
for (NodeTemplateMap vmTopologyMap : helper.getVMTopologyTemplates()) { for (NodeTemplateMap vmTopologyMap : helper.getVMTopologyTemplates()) {
Map<String, Object> provisionedFiles = helper.getNodeArtifact(vmTopologyMap.getNodeTemplate(), "provisioned_files"); Map<String, Object> provisionedFiles = helper.getNodeArtifact(vmTopologyMap.getNodeTemplate(), "provisioned_files");
if (provisionedFiles != null) { if (provisionedFiles != null) {
String fileContentsBase64 = (String) provisionedFiles.get("file_contents"); String encryptedFileContents = (String) provisionedFiles.get("file_contents");
if (fileContentsBase64 != null) { if (encryptedFileContents != null) {
File zipFile = new File(tempInputDir.getParent() + File.separator + Long.toString(System.nanoTime()) + "-" + CLOUD_STORM_FILES_ZIP_SUFIX); File zipFile = new File(tempInputDir.getParent() + File.separator + Long.toString(System.nanoTime()) + "-" + CLOUD_STORM_FILES_ZIP_SUFIX);
String fileContentsBase64 = Converter.decryptString(encryptedFileContents, secret);
Converter.decodeBase64BToFile(fileContentsBase64, zipFile.getAbsolutePath()); Converter.decodeBase64BToFile(fileContentsBase64, zipFile.getAbsolutePath());
Converter.unzipFolder(zipFile.getAbsolutePath(), tempInputDir.getAbsolutePath()); Converter.unzipFolder(zipFile.getAbsolutePath(), tempInputDir.getAbsolutePath());
...@@ -282,6 +301,7 @@ class CloudStormService { ...@@ -282,6 +301,7 @@ class CloudStormService {
int i = 0; int i = 0;
for (NodeTemplateMap vmTopologyMap : vmTopologiesMaps) { for (NodeTemplateMap vmTopologyMap : vmTopologiesMaps) {
Credential toscaCredentials = getHelper().getCredentialsFromVMTopology(vmTopologyMap); Credential toscaCredentials = getHelper().getCredentialsFromVMTopology(vmTopologyMap);
toscaCredentials = Converter.dencryptCredential(toscaCredentials, credentialSecret);
CloudCred cloudStormCredential = new CloudCred(); CloudCred cloudStormCredential = new CloudCred();
cloudStormCredential.setCloudProvider(toscaCredentials.getCloudProviderName()); cloudStormCredential.setCloudProvider(toscaCredentials.getCloudProviderName());
String credInfoFile = toscaCredentials.getCloudProviderName() + i + ".yml"; String credInfoFile = toscaCredentials.getCloudProviderName() + i + ".yml";
...@@ -355,7 +375,7 @@ class CloudStormService { ...@@ -355,7 +375,7 @@ class CloudStormService {
FileUtils.copyDirectory(srcDir, destDir); FileUtils.copyDirectory(srcDir, destDir);
} }
protected ToscaTemplate runCloudStorm(String tempInputDirPath, boolean dryRun) throws IOException, ApiException { protected ToscaTemplate runCloudStorm(String tempInputDirPath, boolean dryRun) throws IOException, ApiException, UnsupportedEncodingException, NoSuchAlgorithmException, NoSuchPaddingException, InvalidKeyException, IllegalBlockSizeException, BadPaddingException {
String[] args = new String[]{"run", tempInputDirPath}; String[] args = new String[]{"run", tempInputDirPath};
File topTopologyFile = new File(tempInputDirPath + TOPOLOGY_RELATIVE_PATH File topTopologyFile = new File(tempInputDirPath + TOPOLOGY_RELATIVE_PATH
+ TOP_TOPOLOGY_FILE_NAME); + TOP_TOPOLOGY_FILE_NAME);
...@@ -402,7 +422,7 @@ class CloudStormService { ...@@ -402,7 +422,7 @@ class CloudStormService {
return keyPair; return keyPair;
} }
protected NodeTemplateMap addCloudStromArtifacts(NodeTemplateMap vmTopologyMap, String tempInputDirPath) throws IOException { protected NodeTemplateMap addCloudStromArtifacts(NodeTemplateMap vmTopologyMap, String tempInputDirPath) throws IOException, UnsupportedEncodingException, NoSuchAlgorithmException, NoSuchPaddingException, InvalidKeyException, IllegalBlockSizeException, BadPaddingException {
Map<String, Object> artifacts = vmTopologyMap.getNodeTemplate().getArtifacts(); Map<String, Object> artifacts = vmTopologyMap.getNodeTemplate().getArtifacts();
if (artifacts == null) { if (artifacts == null) {
artifacts = new HashMap<>(); artifacts = new HashMap<>();
...@@ -416,7 +436,8 @@ class CloudStormService { ...@@ -416,7 +436,8 @@ class CloudStormService {
Logger.getLogger(CloudStormService.class.getName()).log(Level.FINE, "Created zip at: {0}", zipPath); Logger.getLogger(CloudStormService.class.getName()).log(Level.FINE, "Created zip at: {0}", zipPath);
String cloudStormZipFileContentsAsBase64 = Converter.encodeFileToBase64Binary(zipPath); String cloudStormZipFileContentsAsBase64 = Converter.encodeFileToBase64Binary(zipPath);
provisionedFiles.put("file_contents", cloudStormZipFileContentsAsBase64); String encryptedCloudStormZipFileContents = Converter.encryptString(cloudStormZipFileContentsAsBase64, secret);
provisionedFiles.put("file_contents", encryptedCloudStormZipFileContents);
provisionedFiles.put("encoding", "base64"); provisionedFiles.put("encoding", "base64");
provisionedFiles.put("file_ext", "zip"); provisionedFiles.put("file_ext", "zip");
artifacts.put("provisioned_files", provisionedFiles); artifacts.put("provisioned_files", provisionedFiles);
...@@ -425,7 +446,7 @@ class CloudStormService { ...@@ -425,7 +446,7 @@ class CloudStormService {
return vmTopologyMap; return vmTopologyMap;
} }
private void setSSHKeysToVMAttributes(int i, List<NodeTemplateMap> vmTopologiesMaps, CloudsStormSubTopology subTopology, String tempInputDirPath) throws IOException, ApiException { private void setSSHKeysToVMAttributes(int i, List<NodeTemplateMap> vmTopologiesMaps, CloudsStormSubTopology subTopology, String tempInputDirPath) throws IOException, ApiException, UnsupportedEncodingException, NoSuchAlgorithmException, NoSuchPaddingException, InvalidKeyException, IllegalBlockSizeException, BadPaddingException {
NodeTemplateMap vmTopologyMap = vmTopologiesMaps.get(i); NodeTemplateMap vmTopologyMap = vmTopologiesMaps.get(i);
vmTopologyMap = addCloudStromArtifacts(vmTopologyMap, tempInputDirPath); vmTopologyMap = addCloudStromArtifacts(vmTopologyMap, tempInputDirPath);
...@@ -459,7 +480,7 @@ class CloudStormService { ...@@ -459,7 +480,7 @@ class CloudStormService {
userKyes.put("private_key", Converter.encodeFileToBase64Binary(userKyePairFolder + File.separator + "id_rsa")); userKyes.put("private_key", Converter.encodeFileToBase64Binary(userKyePairFolder + File.separator + "id_rsa"));
userKyes.put("public_key", Converter.encodeFileToBase64Binary(userKyePairFolder + File.separator + "id_rsa.pub")); userKyes.put("public_key", Converter.encodeFileToBase64Binary(userKyePairFolder + File.separator + "id_rsa.pub"));
userKeyPairCredential.setKeys(userKyes); userKeyPairCredential.setKeys(userKyes);
// userKeyPairCredential = Converter.encryptCredential(userKeyPairCredential, credentialSecret);
CloudsStormVMs cloudsStormVMs = objectMapper.readValue(new File(tempInputDirPath + TOPOLOGY_RELATIVE_PATH + File.separator + subTopology.getTopology() + ".yml"), CloudsStormVMs cloudsStormVMs = objectMapper.readValue(new File(tempInputDirPath + TOPOLOGY_RELATIVE_PATH + File.separator + subTopology.getTopology() + ".yml"),
CloudsStormVMs.class); CloudsStormVMs.class);
List<CloudsStormVM> vms = cloudsStormVMs.getVms(); List<CloudsStormVM> vms = cloudsStormVMs.getVms();
......
provisioner/src/main/resources/application.properties
View file @ 21ce2a62
...@@ -5,4 +5,6 @@ message.broker.queue.provisioner=provisioner ...@@ -5,4 +5,6 @@ message.broker.queue.provisioner=provisioner
message.broker.queue.planner=planner message.broker.queue.planner=planner
message.broker.queue.deployer=deployer message.broker.queue.deployer=deployer
sure-tosca.base.path=http://localhost:8081/tosca-sure/1.0.0 sure-tosca.base.path=http://localhost:8081/tosca-sure/1.0.0
cloud.storm.secret=top_secret
credential.secret=top_secret
cloud.storm.db.path=etc/UD cloud.storm.db.path=etc/UD
\ No newline at end of file
provisioner/src/test/java/nl/uva/sne/drip/provisioner/CloudStormServiceTest.java
View file @ 21ce2a62
...@@ -284,6 +284,7 @@ public class CloudStormServiceTest { ...@@ -284,6 +284,7 @@ public class CloudStormServiceTest {
assertTrue(provisionedFiles.containsKey("file_contents")); assertTrue(provisionedFiles.containsKey("file_contents"));
String cloudStormZipFileContentsAsBase64 = provisionedFiles.get("file_contents"); String cloudStormZipFileContentsAsBase64 = provisionedFiles.get("file_contents");
cloudStormZipFileContentsAsBase64 = Converter.decryptString(cloudStormZipFileContentsAsBase64, instance.secret);
String testZipPath = tempInputDirPath + File.separator + "test.zip"; String testZipPath = tempInputDirPath + File.separator + "test.zip";
Converter.decodeBase64BToFile(cloudStormZipFileContentsAsBase64, testZipPath); Converter.decodeBase64BToFile(cloudStormZipFileContentsAsBase64, testZipPath);
......
provisioner/src/test/resources/application.properies deleted 100644 → 0
View file @ 0202be19
message.broker.host=127.0.0.1
message.broker.username=guest
message.broker.password=guest
message.broker.queue.provisioner=provisioner
message.broker.queue.planner=planner
message.broker.queue.deployer=deployer
sure-tosca.base.path=http://localhost:8081/tosca-sure/1.0.0
semaphore-python-client-generated/semaphore_client/semaphore_helper.py
View file @ 21ce2a62
import json
from datetime import datetime from datetime import datetime
import urllib3 import urllib3
from semaphore_client import Configuration, ApiClient, api, ProjectRequest, Login, AccessKeyRequest, InventoryRequest, \ from semaphore_client import Configuration, ApiClient, api, ProjectRequest, Login, AccessKeyRequest, InventoryRequest, \
RepositoryRequest, TemplateRequest, Task RepositoryRequest, TemplateRequest, Task, EnvironmentRequest
class SemaphoreHelper: class SemaphoreHelper:
...@@ -77,8 +78,12 @@ class SemaphoreHelper: ...@@ -77,8 +78,12 @@ class SemaphoreHelper:
templates = self.project_api.project_project_id_templates_get(project_id, playbook_name, 'asc') templates = self.project_api.project_project_id_templates_get(project_id, playbook_name, 'asc')
return self.find_template(templates,playbook_name).id return self.find_template(templates,playbook_name).id
def execute_task(self, project_id, template_id, playbook_name): def execute_task(self, project_id, template_id, playbook_name,environment_id=None):
task = Task(template_id=template_id, playbook=playbook_name) environment_json = None
if environment_id:
environment = self.find_environment(project_id,environment_id=environment_id)
environment_json = environment.json
task = Task(template_id=template_id, playbook=playbook_name,environment=environment_json)
self.project_api.project_project_id_tasks_post(task,project_id) self.project_api.project_project_id_tasks_post(task,project_id)
tasks = self.project_api.project_project_id_tasks_get(project_id) tasks = self.project_api.project_project_id_tasks_get(project_id)
return self.find_latest_task(tasks).id return self.find_latest_task(tasks).id
...@@ -127,3 +132,20 @@ class SemaphoreHelper: ...@@ -127,3 +132,20 @@ class SemaphoreHelper:
if repo.git_url == git_url: if repo.git_url == git_url:
return repo return repo
return None return None
def create_environment(self, project_id,name,env_vars):
vars = {"ENV":env_vars}
vars_str = json.dumps(vars)
environment_request = EnvironmentRequest(name=name, project_id=project_id, password=None, json=vars_str)
self.project_api.project_project_id_environment_post(environment_request,project_id)
environment = self.find_environment(project_id, name,vars_str)
return environment.id
def find_environment(self, project_id, name=None,vars_str=None,environment_id=None):
environments = self.project_api.project_project_id_environment_get(project_id,'db-deploy','desc')
for environment in environments:
if environment_id and environment.id ==environment_id:
return environment
if environment.name and environment.name == name and environment.project_id == project_id and environment.json == vars_str:
return environment
return None
\ No newline at end of file
semaphore-python-client-generated/test/test_task.py
View file @ 21ce2a62
...@@ -15,7 +15,7 @@ from __future__ import absolute_import ...@@ -15,7 +15,7 @@ from __future__ import absolute_import
import unittest import unittest
from time import sleep from time import sleep
import datetime
import yaml import yaml
from semaphore_client.semaphore_helper import SemaphoreHelper from semaphore_client.semaphore_helper import SemaphoreHelper
...@@ -30,22 +30,23 @@ class TestTask(unittest.TestCase): ...@@ -30,22 +30,23 @@ class TestTask(unittest.TestCase):
if SemaphoreHelper.service_is_up(self.semaphore_base_url): if SemaphoreHelper.service_is_up(self.semaphore_base_url):
self.username = 'admin' self.username = 'admin'
self.password = 'password' self.password = 'password'
self.project_name = 'test' now = datetime.datetime.now()
self.project_name = 'project_name_test'+'_'+str(now)
self.private_key = '-----BEGIN RSA PRIVATE KEY-----MIIEowIBAAKCAQEAg0blRNV6cm3RTiivpzE8HR4JzKZRVIBZ7bxeNoMz0' \ self.private_key = '-----BEGIN RSA PRIVATE KEY-----MIIEowIBAAKCAQEAg0blRNV6cm3RTiivpzE8HR4JzKZRVIBZ7bxeNoMz0' \
'-----END RSA PRIVATE KEY-----' '-----END RSA PRIVATE KEY-----'
vms = [] vms = []
vm = {} vm = {}
vm['role'] = 'master' vm['role'] = 'master'
vm['public_ip'] = '192.168.1.10' vm['public_ip'] = '127.0.0.1'
vm['user_name'] = 'user_name' vm['semaphore'] = 'semaphore'
vms.append(vm) vms.append(vm)
vm['role'] = 'worker' vm['role'] = 'worker'
vm['public_ip'] = '192.168.1.11' vm['public_ip'] = '127.0.0.1'
vm['role'] = 'worker' vm['role'] = 'worker'
vm['public_ip'] = '192.168.1.12' vm['public_ip'] = '127.0.0.1'
vms.append(vm) vms.append(vm)
self.inventory_contents = yaml.dump( self.build_yml_inventory(vms),default_flow_style=False) self.inventory_contents = yaml.dump( self.build_yml_inventory(vms),default_flow_style=False)
self.git_url = 'https://github.com/skoulouzis/playbooks.git' self.git_url = 'https://github.com/QCDIS/playbooks.git'
self.playbook_name = 'get_ip_addresses.yaml' self.playbook_name = 'get_ip_addresses.yaml'
self.semaphore_helper = SemaphoreHelper(self.semaphore_base_url, self.username, self.password) self.semaphore_helper = SemaphoreHelper(self.semaphore_base_url, self.username, self.password)
...@@ -55,12 +56,22 @@ class TestTask(unittest.TestCase): ...@@ -55,12 +56,22 @@ class TestTask(unittest.TestCase):
def testTask(self): def testTask(self):
if SemaphoreHelper.service_is_up(self.semaphore_base_url): if SemaphoreHelper.service_is_up(self.semaphore_base_url):
project_id = self.semaphore_helper.create_project(self.project_name) project_id = self.semaphore_helper.create_project(self.project_name)
key_id = self.semaphore_helper.create_ssh_key(self.project_name, project_id, self.private_key) now = datetime.datetime.now()
inventory_id = self.semaphore_helper.create_inventory(self.project_name, project_id, key_id, self.inventory_contents) key_name = 'key_name'+str(now)
repository_id = self.semaphore_helper.create_repository(self.project_name, project_id, key_id, self.git_url) key_id = self.semaphore_helper.create_ssh_key(key_name, project_id, self.private_key)
now = datetime.datetime.now()
inventory_name = 'inventory_name'+str(now)
inventory_id = self.semaphore_helper.create_inventory(inventory_name, project_id, key_id, self.inventory_contents)
now = datetime.datetime.now()
repository_name = 'repository_name'+str(now)
repository_id = self.semaphore_helper.create_repository(repository_name, project_id, key_id, self.git_url)
template_id = self.semaphore_helper.create_template(project_id, key_id, inventory_id, repository_id, self.playbook_name) template_id = self.semaphore_helper.create_template(project_id, key_id, inventory_id, repository_id, self.playbook_name)
task_id = self.semaphore_helper.execute_task(project_id, template_id, self.playbook_name) env_vars = {"SOME_VAR": "the contents"}
now = datetime.datetime.now()
environment_name = 'environment_name_name_'+str(now)
environment_id = self.semaphore_helper.create_environment(project_id,environment_name,env_vars)
task_id = self.semaphore_helper.execute_task(project_id, template_id, self.playbook_name,environment_id=environment_id)
for x in range(0, 2): for x in range(0, 2):
task = self.semaphore_helper.get_task(project_id, task_id) task = self.semaphore_helper.get_task(project_id, task_id)
print(task) print(task)
...@@ -68,20 +79,20 @@ class TestTask(unittest.TestCase): ...@@ -68,20 +79,20 @@ class TestTask(unittest.TestCase):
# print(task_output) # print(task_output)
sleep(1) sleep(1)
task_id = self.semaphore_helper.execute_task(project_id, template_id, self.playbook_name) # task_id = self.semaphore_helper.execute_task(project_id, template_id, self.playbook_name)
for x in range(0, 2): # for x in range(0, 2):
task = self.semaphore_helper.get_task(project_id, task_id) # task = self.semaphore_helper.get_task(project_id, task_id)
print(task) # print(task)
# task_output = self.semaphore_helper.get_task_output(project_id, task_id) # # task_output = self.semaphore_helper.get_task_output(project_id, task_id)
# print(task_output) # # print(task_output)
sleep(1) # sleep(1)
task_id = self.semaphore_helper.execute_task(project_id, template_id, 'mount.yaml') # task_id = self.semaphore_helper.execute_task(project_id, template_id, 'get_ip_addresses.yaml')
for x in range(0, 2): # for x in range(0, 2):
task = self.semaphore_helper.get_task(project_id, task_id) # task = self.semaphore_helper.get_task(project_id, task_id)
print(task) # print(task)
# task_output = self.semaphore_helper.get_task_output(project_id, task_id) # # task_output = self.semaphore_helper.get_task_output(project_id, task_id)
# print(task_output) # # print(task_output)
sleep(1) # sleep(1)
...@@ -89,7 +100,7 @@ class TestTask(unittest.TestCase): ...@@ -89,7 +100,7 @@ class TestTask(unittest.TestCase):
inventory = {} inventory = {}
all = {} all = {}
vars = {'ansible_ssh_common_args':'-o StrictHostKeyChecking=no'} vars = {'ansible_ssh_common_args':'-o StrictHostKeyChecking=no'}
vars['ansible_ssh_user'] = vms[0]['user_name'] vars['ansible_ssh_user'] = vms[0]['semaphore']
children = {} children = {}
for vm in vms: for vm in vms:
role = vm['role'] role = vm['role']
......
sure_tosca-client_python_stubs/test/test_default_api.py
View file @ 21ce2a62
...@@ -14,9 +14,10 @@ from __future__ import absolute_import ...@@ -14,9 +14,10 @@ from __future__ import absolute_import
import os import os
import unittest import unittest
from urllib.error import HTTPError
import urllib3 from urllib.request import urlopen
import ssl
import sure_tosca_client import sure_tosca_client
from sure_tosca_client import Configuration, ApiClient from sure_tosca_client import Configuration, ApiClient
from sure_tosca_client.api.default_api import DefaultApi # noqa: E501 from sure_tosca_client.api.default_api import DefaultApi # noqa: E501
...@@ -28,7 +29,7 @@ class TestDefaultApi(unittest.TestCase): ...@@ -28,7 +29,7 @@ class TestDefaultApi(unittest.TestCase):
def setUp(self): def setUp(self):
configuration = Configuration() configuration = Configuration()
configuration.host = 'http://127.0.0.1:8081/tosca-sure/1.0.0' #Make sure we don't have '/' on the end of url configuration.host = 'http://127.0.0.1:8081/tosca-sure/1.0.0' #Make sure we don't have '/' on the end of url
if self.service_is_up(configuration.host): if self.check_service_is_up(configuration.host):
configuration.verify_ssl = False configuration.verify_ssl = False
api_client = ApiClient(configuration=configuration) api_client = ApiClient(configuration=configuration)
self.api = sure_tosca_client.api.default_api.DefaultApi(api_client=api_client) # noqa: E501 self.api = sure_tosca_client.api.default_api.DefaultApi(api_client=api_client) # noqa: E501
...@@ -190,12 +191,17 @@ class TestDefaultApi(unittest.TestCase): ...@@ -190,12 +191,17 @@ class TestDefaultApi(unittest.TestCase):
return file_id return file_id
def service_is_up(self, url): def check_service_is_up(self, url):
code = None code = None
try: try:
http = urllib3.PoolManager() response = urlopen(url, context=ssl._create_unverified_context())
r = http.request('HEAD', url) print(response)
# http = urllib3.PoolManager()
# r = http.request('HEAD', url,context=ctx)
except Exception as e: except Exception as e:
if isinstance(e, HTTPError) and e.code == 401:
return True
return False return False
return True return True
......
sure_tosca-flask-server/.dockerignore
View file @ 21ce2a62
...@@ -3,7 +3,7 @@ ...@@ -3,7 +3,7 @@
README.md README.md
tox.ini tox.ini
git_push.sh git_push.sh
test-requirements.txt #test-requirements.txt
setup.py setup.py
# Byte-compiled / optimized / DLL files # Byte-compiled / optimized / DLL files
......
sure_tosca-flask-server/Dockerfile
View file @ 21ce2a62
FROM python:3.7-buster FROM python:3.6-buster
RUN mkdir -p /usr/src/app RUN mkdir -p /usr/src/app
WORKDIR /usr/src/app WORKDIR /usr/src/app
COPY requirements.txt /usr/src/app/ COPY requirements.txt /usr/src/app/
COPY test-requirements.txt /usr/src/app/
RUN pip install --no-cache-dir -r requirements.txt RUN pip install --no-cache-dir -r requirements.txt
#RUN pip install --no-cache-dir -r test-requirements.txt
COPY . /usr/src/app COPY . /usr/src/app
#RUN python -m unittest discover
#RUN pip uninstall -y --no-cache-dir -r test-requirements.txt
EXPOSE 8081 EXPOSE 8081
ENTRYPOINT ["python"] ENTRYPOINT ["python"]
......
sure_tosca-flask-server/requirements.txt
View file @ 21ce2a62
...@@ -6,3 +6,4 @@ swagger-ui-bundle==0.0.6 ...@@ -6,3 +6,4 @@ swagger-ui-bundle==0.0.6
tinydb==3.15.2 tinydb==3.15.2
tosca-parser==2.0.0 tosca-parser==2.0.0
flask-restx==0.2.0 flask-restx==0.2.0
werkzeug==1.0.1
\ No newline at end of file
sure_tosca-flask-server/sure_tosca/service/tosca_template_service.py
View file @ 21ce2a62
import json import copy
import logging import logging
import os import os
import tempfile import tempfile
import time import time
import uuid
from builtins import print
from functools import reduce from functools import reduce
import copy
import yaml import yaml
from tinydb import TinyDB, Query from tinydb import TinyDB, Query
from tinydb.middlewares import CachingMiddleware from tinydb.middlewares import CachingMiddleware
from tinydb.storages import MemoryStorage from tinydb.storages import MemoryStorage
from toscaparser.functions import GetAttribute
from toscaparser.tosca_template import ToscaTemplate from toscaparser.tosca_template import ToscaTemplate
from werkzeug.datastructures import FileStorage from werkzeug.datastructures import FileStorage
...@@ -20,7 +16,6 @@ from sure_tosca.models.base_model_ import Model ...@@ -20,7 +16,6 @@ from sure_tosca.models.base_model_ import Model
from sure_tosca.models.node_template import NodeTemplateModel as NodeTemplateModel from sure_tosca.models.node_template import NodeTemplateModel as NodeTemplateModel
from sure_tosca.models.node_template_map import NodeTemplateMapModel from sure_tosca.models.node_template_map import NodeTemplateMapModel
from sure_tosca.models.tosca_template import ToscaTemplateModel as ToscaTemplateModel from sure_tosca.models.tosca_template import ToscaTemplateModel as ToscaTemplateModel
from sure_tosca.service import tosca_helper from sure_tosca.service import tosca_helper
# db = TinyDB(storage=CachingMiddleware(MemoryStorage)) # db = TinyDB(storage=CachingMiddleware(MemoryStorage))
...@@ -110,7 +105,7 @@ def save(file: FileStorage): ...@@ -110,7 +105,7 @@ def save(file: FileStorage):
purge_all_tables() purge_all_tables()
dictionary = yaml.safe_load(file.stream) dictionary = yaml.safe_load(file.stream)
# dictionary = yaml.load(file.stream) # dictionary = yaml.load(file.stream)
logger.info("tosca template: \n" + str(yaml.dump(dictionary))) logger.debug("tosca template: \n" + str(yaml.dump(dictionary)))
# print(yaml.dump(dictionary)) # print(yaml.dump(dictionary))
tosca_template = get_tosca_template(dictionary) tosca_template = get_tosca_template(dictionary)
# all_custom_def = tosca_template.nodetemplates[0].custom_def # all_custom_def = tosca_template.nodetemplates[0].custom_def
......
sure_tosca-flask-server/sure_tosca/service/tosca_type_service.py
View file @ 21ce2a62
from sure_tosca.models.node_template import NodeTemplateModel # noqa: E501
from sure_tosca.models.node_template_map import NodeTemplateMapModel # noqa: E501
from sure_tosca.models.tosca_template import ToscaTemplateModel # noqa: E501
def add_type_definition(definition_map): def add_type_definition(definition_map):
return None return None
......
sure_tosca-flask-server/sure_tosca/test/test_default_controller.py
View file @ 21ce2a62
...@@ -2,25 +2,59 @@ ...@@ -2,25 +2,59 @@
from __future__ import absolute_import from __future__ import absolute_import
import os
import tempfile
import requests
from flask import json from flask import json
from six import BytesIO from six import BytesIO
import os
from sure_tosca.models.node_template import NodeTemplateModel # noqa: E501
from sure_tosca.models.topology_template import TopologyTemplateModel # noqa: E501
from sure_tosca.models.tosca_template import ToscaTemplateModel # noqa: E501
from sure_tosca.test import BaseTestCase from sure_tosca.test import BaseTestCase
class TestDefaultController(BaseTestCase): class TestDefaultController(BaseTestCase):
"""DefaultController integration test stubs""" """DefaultController integration test stubs"""
def test_upload(self):
"""Test case for get_all_ancestor_properties
"""
id_example = self.upload_file('https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/TIC.yaml')
doc_id = int(id_example)
self.assertIsInstance(doc_id,int)
# id_example = self.upload_file('https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/application_example_planed.yaml')
# doc_id = int(id_example)
self.assertIsInstance(doc_id,int)
id_example = self.upload_file(
'https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/application_example_provisioned.yaml')
doc_id = int(id_example)
self.assertIsInstance(doc_id,int)
id_example = self.upload_file(
'https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/glusterFS.yaml')
doc_id = int(id_example)
self.assertIsInstance(doc_id,int)
id_example = self.upload_file(
'https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/kubernetes.yaml')
doc_id = int(id_example)
self.assertIsInstance(doc_id,int)
id_example = self.upload_file(
'https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/lifeWatch_vre1.yaml')
doc_id = int(id_example)
self.assertIsInstance(doc_id,int)
id_example = self.upload_file(
'https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/topology.yaml')
doc_id = int(id_example)
self.assertIsInstance(doc_id,int)
def test_get_all_ancestor_properties(self): def test_get_all_ancestor_properties(self):
"""Test case for get_all_ancestor_properties """Test case for get_all_ancestor_properties
""" """
id_example = self.upload_file('application_example_2_topologies.yaml') id_example = self.upload_file('https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/application_example_2_topologies.yaml')
self.assertTrue(id_example.strip().isdigit()) self.assertTrue(id_example.strip().isdigit())
response = self.client.open( response = self.client.open(
'/tosca-sure/1.0.0/tosca_template/{id}/topology_template/node_templates/{node_name}/ancestors_properties'.format( '/tosca-sure/1.0.0/tosca_template/{id}/topology_template/node_templates/{node_name}/ancestors_properties'.format(
...@@ -35,7 +69,7 @@ class TestDefaultController(BaseTestCase): ...@@ -35,7 +69,7 @@ class TestDefaultController(BaseTestCase):
""" """
id_example = self.upload_file('application_example_2_topologies.yaml') id_example = self.upload_file('https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/application_example_2_topologies.yaml')
self.assertTrue(id_example.strip().isdigit()) self.assertTrue(id_example.strip().isdigit())
response = self.client.open( response = self.client.open(
'/tosca-sure/1.0.0/tosca_template/{id}/topology_template/node_templates/{node_name}/ancestors_types'.format( '/tosca-sure/1.0.0/tosca_template/{id}/topology_template/node_templates/{node_name}/ancestors_types'.format(
...@@ -50,7 +84,7 @@ class TestDefaultController(BaseTestCase): ...@@ -50,7 +84,7 @@ class TestDefaultController(BaseTestCase):
""" """
id_example = self.upload_file('application_example_2_topologies.yaml') id_example = self.upload_file('https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/application_example_2_topologies.yaml')
self.assertTrue(id_example.strip().isdigit()) self.assertTrue(id_example.strip().isdigit())
response = self.client.open( response = self.client.open(
'/tosca-sure/1.0.0/tosca_template/{id}/topology_template/node_templates/{node_name}/ancestors_requirements'.format( '/tosca-sure/1.0.0/tosca_template/{id}/topology_template/node_templates/{node_name}/ancestors_requirements'.format(
...@@ -66,7 +100,7 @@ class TestDefaultController(BaseTestCase): ...@@ -66,7 +100,7 @@ class TestDefaultController(BaseTestCase):
""" """
# query_string = [('anchors', 'anchors_example'), ('derived_from', 'derived_from_example')] # query_string = [('anchors', 'anchors_example'), ('derived_from', 'derived_from_example')]
id_example = self.upload_file('application_example_2_topologies.yaml') id_example = self.upload_file('https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/application_example_2_topologies.yaml')
self.assertTrue(id_example.strip().isdigit()) self.assertTrue(id_example.strip().isdigit())
response = self.client.open( response = self.client.open(
'/tosca-sure/1.0.0/tosca_template/{id}/dsl_definitions'.format(id=id_example), '/tosca-sure/1.0.0/tosca_template/{id}/dsl_definitions'.format(id=id_example),
...@@ -78,7 +112,7 @@ class TestDefaultController(BaseTestCase): ...@@ -78,7 +112,7 @@ class TestDefaultController(BaseTestCase):
""" """
id_example = self.upload_file('application_example_2_topologies.yaml') id_example = self.upload_file('https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/application_example_2_topologies.yaml')
self.assertTrue(id_example.strip().isdigit()) self.assertTrue(id_example.strip().isdigit())
response = self.client.open( response = self.client.open(
'/tosca-sure/1.0.0/tosca_template/{id}/imports'.format(id=id_example), '/tosca-sure/1.0.0/tosca_template/{id}/imports'.format(id=id_example),
...@@ -90,7 +124,7 @@ class TestDefaultController(BaseTestCase): ...@@ -90,7 +124,7 @@ class TestDefaultController(BaseTestCase):
""" """
id_example = self.upload_file('application_example_outputs.yaml') id_example = self.upload_file('https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/application_example_outputs.yaml')
self.assertTrue(id_example.strip().isdigit()) self.assertTrue(id_example.strip().isdigit())
response = self.client.open( response = self.client.open(
'/tosca-sure/1.0.0/tosca_template/{id}/topology_template/node_templates/{node_name}/outputs'.format( '/tosca-sure/1.0.0/tosca_template/{id}/topology_template/node_templates/{node_name}/outputs'.format(
...@@ -105,7 +139,7 @@ class TestDefaultController(BaseTestCase): ...@@ -105,7 +139,7 @@ class TestDefaultController(BaseTestCase):
""" """
id_example = self.upload_file('application_example_2_topologies.yaml') id_example = self.upload_file('https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/application_example_2_topologies.yaml')
self.assertTrue(id_example.strip().isdigit()) self.assertTrue(id_example.strip().isdigit())
response = self.client.open( response = self.client.open(
'/tosca-sure/1.0.0/tosca_template/{id}/topology_template/node_templates/{node_name}/properties'.format( '/tosca-sure/1.0.0/tosca_template/{id}/topology_template/node_templates/{node_name}/properties'.format(
...@@ -120,7 +154,7 @@ class TestDefaultController(BaseTestCase): ...@@ -120,7 +154,7 @@ class TestDefaultController(BaseTestCase):
""" """
id_example = self.upload_file('application_example_2_topologies.yaml') id_example = self.upload_file('https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/application_example_2_topologies.yaml')
self.assertTrue(id_example.strip().isdigit()) self.assertTrue(id_example.strip().isdigit())
response = self.client.open( response = self.client.open(
'/tosca-sure/1.0.0/tosca_template/{id}/topology_template/node_templates/{node_name}/requirements'.format( '/tosca-sure/1.0.0/tosca_template/{id}/topology_template/node_templates/{node_name}/requirements'.format(
...@@ -135,7 +169,7 @@ class TestDefaultController(BaseTestCase): ...@@ -135,7 +169,7 @@ class TestDefaultController(BaseTestCase):
""" """
id_example = self.upload_file('application_example_2_topologies.yaml') id_example = self.upload_file('https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/application_example_2_topologies.yaml')
self.assertTrue(id_example.strip().isdigit()) self.assertTrue(id_example.strip().isdigit())
query_string = [('type_name', None), query_string = [('type_name', None),
('node_name', 'compute'), ('node_name', 'compute'),
...@@ -206,7 +240,7 @@ class TestDefaultController(BaseTestCase): ...@@ -206,7 +240,7 @@ class TestDefaultController(BaseTestCase):
""" """
id_example = self.upload_file('application_example_2_topologies.yaml') id_example = self.upload_file('https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/application_example_2_topologies.yaml')
self.assertTrue(id_example.strip().isdigit()) self.assertTrue(id_example.strip().isdigit())
response = self.client.open( response = self.client.open(
'/tosca-sure/1.0.0/tosca_template/{id}/topology_template/node_templates/{node_name}/type_name'.format( '/tosca-sure/1.0.0/tosca_template/{id}/topology_template/node_templates/{node_name}/type_name'.format(
...@@ -221,7 +255,7 @@ class TestDefaultController(BaseTestCase): ...@@ -221,7 +255,7 @@ class TestDefaultController(BaseTestCase):
""" """
id_example = self.upload_file('application_example_2_topologies.yaml') id_example = self.upload_file('https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/application_example_2_topologies.yaml')
self.assertTrue(id_example.strip().isdigit()) self.assertTrue(id_example.strip().isdigit())
response = self.client.open( response = self.client.open(
'/tosca-sure/1.0.0/tosca_template/{id}/topology_template/node_templates/{node_name}/derived_from'.format( '/tosca-sure/1.0.0/tosca_template/{id}/topology_template/node_templates/{node_name}/derived_from'.format(
...@@ -236,7 +270,7 @@ class TestDefaultController(BaseTestCase): ...@@ -236,7 +270,7 @@ class TestDefaultController(BaseTestCase):
""" """
id_example = self.upload_file('application_example_2_topologies.yaml') id_example = self.upload_file('https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/application_example_2_topologies.yaml')
self.assertTrue(id_example.strip().isdigit()) self.assertTrue(id_example.strip().isdigit())
response = self.client.open( response = self.client.open(
'/tosca-sure/1.0.0/tosca_template/{id}/topology_template/node_templates/{node_name}/related'.format( '/tosca-sure/1.0.0/tosca_template/{id}/topology_template/node_templates/{node_name}/related'.format(
...@@ -251,7 +285,7 @@ class TestDefaultController(BaseTestCase): ...@@ -251,7 +285,7 @@ class TestDefaultController(BaseTestCase):
""" """
id_example = self.upload_file('application_example_2_topologies.yaml') id_example = self.upload_file('https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/application_example_2_topologies.yaml')
self.assertTrue(id_example.strip().isdigit()) self.assertTrue(id_example.strip().isdigit())
query_string = [('type_name', None), query_string = [('type_name', None),
('derived_from', None)] ('derived_from', None)]
...@@ -266,7 +300,7 @@ class TestDefaultController(BaseTestCase): ...@@ -266,7 +300,7 @@ class TestDefaultController(BaseTestCase):
""" """
id_example = self.upload_file('application_example_2_topologies.yaml') id_example = self.upload_file('https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/application_example_2_topologies.yaml')
self.assertTrue(id_example.strip().isdigit()) self.assertTrue(id_example.strip().isdigit())
response = self.client.open( response = self.client.open(
'/tosca-sure/1.0.0/tosca_template/{id}/topology_template'.format(id=id_example), '/tosca-sure/1.0.0/tosca_template/{id}/topology_template'.format(id=id_example),
...@@ -280,7 +314,7 @@ class TestDefaultController(BaseTestCase): ...@@ -280,7 +314,7 @@ class TestDefaultController(BaseTestCase):
""" """
id_example = self.upload_file('application_example_2_topologies.yaml') id_example = self.upload_file('https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/application_example_2_topologies.yaml')
self.assertTrue(id_example.strip().isdigit()) self.assertTrue(id_example.strip().isdigit())
response = self.client.open( response = self.client.open(
'/tosca-sure/1.0.0/tosca_template/{id}'.format(id=id_example), '/tosca-sure/1.0.0/tosca_template/{id}'.format(id=id_example),
...@@ -294,7 +328,7 @@ class TestDefaultController(BaseTestCase): ...@@ -294,7 +328,7 @@ class TestDefaultController(BaseTestCase):
""" """
id_example = self.upload_file('application_example_2_topologies.yaml') id_example = self.upload_file('https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/application_example_2_topologies.yaml')
self.assertTrue(id_example.strip().isdigit()) self.assertTrue(id_example.strip().isdigit())
query_string = [('kind_of_type', 'interface_types'), query_string = [('kind_of_type', 'interface_types'),
('has_interfaces', None), ('has_interfaces', None),
...@@ -334,7 +368,7 @@ class TestDefaultController(BaseTestCase): ...@@ -334,7 +368,7 @@ class TestDefaultController(BaseTestCase):
""" """
id_example = self.upload_file('application_example_2_topologies.yaml') id_example = self.upload_file('https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/application_example_2_topologies.yaml')
self.assertTrue(id_example.strip().isdigit()) self.assertTrue(id_example.strip().isdigit())
properties = {'properties': {'cpu_frequency': '2 GHz'}} properties = {'properties': {'cpu_frequency': '2 GHz'}}
response = self.client.open( response = self.client.open(
...@@ -354,7 +388,7 @@ class TestDefaultController(BaseTestCase): ...@@ -354,7 +388,7 @@ class TestDefaultController(BaseTestCase):
""" """
id_example = self.upload_file('application_example_2_topologies.yaml') id_example = self.upload_file('https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/application_example_2_topologies.yaml')
self.assertTrue(id_example.strip().isdigit()) self.assertTrue(id_example.strip().isdigit())
response = self.client.open( response = self.client.open(
'/tosca-sure/1.0.0/tosca_template/{id}/topology_template/node_templates'.format(id=id_example), '/tosca-sure/1.0.0/tosca_template/{id}/topology_template/node_templates'.format(id=id_example),
...@@ -370,7 +404,7 @@ class TestDefaultController(BaseTestCase): ...@@ -370,7 +404,7 @@ class TestDefaultController(BaseTestCase):
""" """
id_example = self.upload_file('application_example_2_topologies.yaml') id_example = self.upload_file('https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/application_example_2_topologies.yaml')
self.assertTrue(id_example.strip().isdigit()) self.assertTrue(id_example.strip().isdigit())
query_string = [('instance_name', 'instance_name_example'), query_string = [('instance_name', 'instance_name_example'),
('operation_name', 'provision')] ('operation_name', 'provision')]
...@@ -383,13 +417,8 @@ class TestDefaultController(BaseTestCase): ...@@ -383,13 +417,8 @@ class TestDefaultController(BaseTestCase):
self.assertEqual(response.status_code, 200) self.assertEqual(response.status_code, 200)
self.assertIsInstance(response.json, dict) self.assertIsInstance(response.json, dict)
def upload_file(self, file_name): def upload_file(self, url):
tosca_path = "../../../TOSCA/" input_tosca_file_path = self.get_remote_tosca_file(url)
input_tosca_file_path = tosca_path + '/' + file_name
if not os.path.exists(input_tosca_file_path):
tosca_path = "../TOSCA/"
input_tosca_file_path = tosca_path + '/' + file_name
dir_path = os.path.dirname(os.path.realpath(__file__)) dir_path = os.path.dirname(os.path.realpath(__file__))
self.assertEqual(True, os.path.exists(input_tosca_file_path), self.assertEqual(True, os.path.exists(input_tosca_file_path),
'Starting from: ' + dir_path + ' Input TOSCA file: ' + input_tosca_file_path + ' not found') 'Starting from: ' + dir_path + ' Input TOSCA file: ' + input_tosca_file_path + ' not found')
...@@ -405,6 +434,11 @@ class TestDefaultController(BaseTestCase): ...@@ -405,6 +434,11 @@ class TestDefaultController(BaseTestCase):
content_type='multipart/form-data') content_type='multipart/form-data')
return response.data.decode('utf-8').replace('\n', '') return response.data.decode('utf-8').replace('\n', '')
def get_remote_tosca_file(self, url):
tosca = requests.get(url)
input_tosca_file_path = os.path.join(tempfile.gettempdir(),'test_tosca_file.yaml')
open( input_tosca_file_path, 'wb').write(tosca.content)
return input_tosca_file_path
if __name__ == '__main__': if __name__ == '__main__':
import unittest import unittest
......
sure_tosca-flask-server/sure_tosca/test/test_tosca_template_service.py
View file @ 21ce2a62
import os import os
import tempfile
from unittest import TestCase from unittest import TestCase
import requests
from six import BytesIO from six import BytesIO
from werkzeug.datastructures import FileStorage from werkzeug.datastructures import FileStorage
import logging import logging
...@@ -15,16 +18,18 @@ formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(messag ...@@ -15,16 +18,18 @@ formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(messag
h.setFormatter(formatter) h.setFormatter(formatter)
logger.addHandler(h) logger.addHandler(h)
logger.handler_set = True logger.handler_set = True
base_url = 'https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/'
class Test(TestCase): class Test(TestCase):
def test_upload(self): def test_upload(self):
file_names = ['application_example_outputs.yaml','application_example_2_topologies.yaml','lifeWatch_vre1.yaml', file_names = ['application_example_outputs.yaml','application_example_2_topologies.yaml','lifeWatch_vre1.yaml',
'application_example_updated.yaml','compute.yaml','application_example_provisioned.yaml','topology.yaml', 'application_example_updated.yaml','compute.yaml','application_example_provisioned.yaml','topology.yaml',
'kubernetes.yaml','application_example_planed.yaml'] 'kubernetes.yaml','TIC.yaml']
for file_name in file_names: for file_name in file_names:
logger.info("Testing : " + str(file_name)) logger.info("Testing : " + str(file_name))
doc_id = tosca_template_service.save(self.upload_file(file_name)) doc_id = tosca_template_service.save(self.upload_file(base_url+file_name))
logger.info("doc_id : " + str(doc_id)) logger.info("doc_id : " + str(doc_id))
self.assertTrue (doc_id >= 0) self.assertTrue (doc_id >= 0)
...@@ -39,7 +44,7 @@ class Test(TestCase): ...@@ -39,7 +44,7 @@ class Test(TestCase):
def test_get_tosca_template_model_by_id(self): def test_get_tosca_template_model_by_id(self):
doc_id = tosca_template_service.save(self.upload_file('application_example_updated.yaml')) doc_id = tosca_template_service.save(self.upload_file(base_url+'application_example_updated.yaml'))
tosca_template_dict = tosca_template_service.get_tosca_template_dict_by_id(doc_id) tosca_template_dict = tosca_template_service.get_tosca_template_dict_by_id(doc_id)
tosca_template_model = ToscaTemplateModel.from_dict(tosca_template_dict) tosca_template_model = ToscaTemplateModel.from_dict(tosca_template_dict)
...@@ -47,17 +52,12 @@ class Test(TestCase): ...@@ -47,17 +52,12 @@ class Test(TestCase):
self.assertIsNotNone(tosca_template_model.topology_template) self.assertIsNotNone(tosca_template_model.topology_template)
self.assertIsNotNone(tosca_template_model.topology_template.node_templates) self.assertIsNotNone(tosca_template_model.topology_template.node_templates)
def upload_file(self, file_name):
tosca_path = "../../../TOSCA/"
input_tosca_file_path = tosca_path + '/' + file_name
if not os.path.exists(input_tosca_file_path):
tosca_path = "../TOSCA/"
input_tosca_file_path = tosca_path + '/' + file_name
def upload_file(self, url):
input_tosca_file_path = self.get_remote_tosca_file(url)
dir_path = os.path.dirname(os.path.realpath(__file__)) dir_path = os.path.dirname(os.path.realpath(__file__))
self.assertEqual(True, os.path.exists(input_tosca_file_path), self.assertEqual(True, os.path.exists(input_tosca_file_path),
'Starting from: ' + dir_path + ' Input TOSCA file: ' + input_tosca_file_path + ' not found') 'Starting from: ' + dir_path + ' Input TOSCA file: ' + input_tosca_file_path + ' not found')
file = open(input_tosca_file_path, "r") file = open(input_tosca_file_path, "r")
# with open(input_tosca_file_path, 'r') as file: # with open(input_tosca_file_path, 'r') as file:
# contents = file.read() # contents = file.read()
...@@ -71,3 +71,10 @@ class Test(TestCase): ...@@ -71,3 +71,10 @@ class Test(TestCase):
return FileStorage(stream=BytesIO(byte_contents), filename=input_tosca_file_path, name=input_tosca_file_path, return FileStorage(stream=BytesIO(byte_contents), filename=input_tosca_file_path, name=input_tosca_file_path,
content_type=None, content_type=None,
content_length=None, headers=None) content_length=None, headers=None)
def get_remote_tosca_file(self, url):
tosca = requests.get(url)
input_tosca_file_path = os.path.join(tempfile.gettempdir(),'test_tosca_file.yaml')
open( input_tosca_file_path, 'wb').write(tosca.content)
return input_tosca_file_path
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment