Skip to content

C
CONF
Commit 21ce2a62 authored by Alfonso Orta's avatar Alfonso Orta
Browse files
Options

Merge branch 'develop' into 'staging' 

Develop

See merge request !3
parents 0202be19 ea18c6db
Show whitespace changes
Inline Side-by-side
Showing with 857 additions and 315 deletions
bin/docker_push.sh
View file @ 21ce2a62
#!/bin/bash
docker push qcdis/manager:3.0.0
docker push qcdis/provisioner:3.0.0
# docker push qcdis/sure-tosca:3.0.0
# docker push qcdis/planner:3.0.0
# docker push qcdis/deployer:3.0.0
docker push qcdis/manager
docker push qcdis/provisioner
docker push qcdis/sure-tosca
docker push qcdis/planner
docker push qcdis/deployer
bin/dockerize.sh
View file @ 21ce2a62
......@@ -10,21 +10,21 @@ cd provisioner && mvn -Dmaven.test.skip=true dockerfile:build
[ $status -eq 0 ] && echo "build successful" || exit -1
cd ../
echo "----------Building sure_tosca-flask-server Docker--------------"
cd sure_tosca-flask-server && docker build -t sure-tosca:3.0.0 .
cd sure_tosca-flask-server && docker build -t sure-tosca .
[ $status -eq 0 ] && echo "build successful" || exit -1
echo "----------Building planner Docker--------------"
cd ../
cd planner && docker build -t planner:3.0.0 .
cd planner && docker build -t planner .
[ $status -eq 0 ] && echo "build successful" || exit -1
echo "----------Building deployer Docker--------------"
cd ../
cd deployer && docker build -t deployer:3.0.0 .
cd deployer && docker build -t deployer .
[ $status -eq 0 ] && echo "build successful" || exit -1
docker tag sure-tosca:3.0.0 qcdis/sure-tosca:3.0.0
docker tag planner:3.0.0 qcdis/planner:3.0.0
docker tag deployer:3.0.0 qcdis/deployer:3.0.0
docker tag manager:3.0.0 qcdis/manager:3.0.0
docker tag provisioner:3.0.0 qcdis/provisioner:3.0.0
docker tag sure-tosca qcdis/sure-tosca
docker tag planner qcdis/planner
docker tag deployer qcdis/deployer
docker tag manager:3.0.0 qcdis/manager
docker tag provisioner:3.0.0 qcdis/provisioner
commons/src/main/java/nl/uva/sne/drip/commons/utils/Converter.java
View file @ 21ce2a62
......@@ -21,6 +21,7 @@ import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;
import java.nio.charset.StandardCharsets;
import java.nio.file.FileSystem;
import java.nio.file.FileSystems;
......@@ -30,16 +31,26 @@ import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.security.InvalidKeyException;
import java.security.Key;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.Arrays;
import java.util.Base64;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
import java.util.zip.ZipOutputStream;
import javax.crypto.BadPaddingException;
import javax.crypto.Cipher;
import javax.crypto.IllegalBlockSizeException;
import javax.crypto.NoSuchPaddingException;
import javax.crypto.spec.SecretKeySpec;
import nl.uva.sne.drip.model.tosca.Credential;
import org.json.JSONException;
import org.json.JSONObject;
import org.springframework.web.multipart.MultipartFile;
......@@ -157,4 +168,62 @@ public class Converter {
}
}
public static String decryptString(String contents, String secret) throws UnsupportedEncodingException, NoSuchAlgorithmException, NoSuchPaddingException, InvalidKeyException, IllegalBlockSizeException, BadPaddingException {
SecretKeySpec secretKey = getsecretKey(secret);
Cipher cipher = Cipher.getInstance("AES/ECB/PKCS5PADDING");
cipher.init(Cipher.DECRYPT_MODE, secretKey);
return new String(cipher.doFinal(Base64.getDecoder().decode(contents)));
}
public static String encryptString(String contents, String secret) throws UnsupportedEncodingException, NoSuchAlgorithmException, NoSuchPaddingException, InvalidKeyException, IllegalBlockSizeException, BadPaddingException {
SecretKeySpec secretKey = getsecretKey(secret);
Cipher cipher = Cipher.getInstance("AES/ECB/PKCS5Padding");
cipher.init(Cipher.ENCRYPT_MODE, secretKey);
return Base64.getEncoder().encodeToString(cipher.doFinal(contents.getBytes("UTF-8")));
}
private static SecretKeySpec getsecretKey(String myKey) throws UnsupportedEncodingException, NoSuchAlgorithmException {
MessageDigest sha;
byte[] key = myKey.getBytes("UTF-8");
sha = MessageDigest.getInstance("SHA-1");
key = sha.digest(key);
key = Arrays.copyOf(key, 16);
return new SecretKeySpec(key, "AES");
}
public static Credential encryptCredential(Credential credential, String credentialSecret) throws UnsupportedEncodingException, NoSuchAlgorithmException, NoSuchPaddingException, InvalidKeyException, IllegalBlockSizeException, BadPaddingException {
Map<String, String> credKeys = credential.getKeys();
Set<String> keySet = credKeys.keySet();
for (String key : keySet) {
String credKey = credKeys.get(key);
if (credKey != null) {
credKeys.put(key, encryptString(credKey, credentialSecret));
}
}
String token = credential.getToken();
if (token != null) {
credential.setToken(encryptString(token, credentialSecret));
}
return credential;
}
public static Credential dencryptCredential(Credential credential, String credentialSecret) throws UnsupportedEncodingException, NoSuchAlgorithmException, NoSuchPaddingException, InvalidKeyException, IllegalBlockSizeException, BadPaddingException {
Map<String, String> credKeys = credential.getKeys();
Set<String> keySet = credKeys.keySet();
for (String key : keySet) {
String credKey = credKeys.get(key);
if (credKey != null) {
credKeys.put(key, decryptString(credKey, credentialSecret));
}
}
String token = credential.getToken();
if (token != null) {
credential.setToken(decryptString(token, credentialSecret));
}
return credential;
}
}
commons/src/main/resources/application.properties deleted 100644 → 0
View file @ 0202be19
message.broker.host=127.0.0.1
message.broker.username=guest
message.broker.password=guest
message.broker.queue.provisioner=provisioner
message.broker.queue.planner=planner
message.broker.queue.deployer=deployer
sure-tosca.base.path=http://localhost:8081/tosca-sure/1.0.0
commons/src/test/java/nl/uva/sne/drip/commons/utils/ConverterTest.java 0 → 100644
View file @ 21ce2a62
/*
* Copyright 2019 S. Koulouzis
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nl.uva.sne.drip.commons.utils;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import static org.junit.Assert.*;
/**
*
* @author S. Koulouzis
*/
public class ConverterTest {
public ConverterTest() {
}
@BeforeClass
public static void setUpClass() {
}
@AfterClass
public static void tearDownClass() {
}
@Before
public void setUp() {
}
@After
public void tearDown() {
}
/**
* Test of decryptString method, of class Converter.
*/
@Test
public void testEncryptDecryptString() throws Exception {
System.out.println("decryptString");
String contents = "this is very important information";
String secret = "top_secret";
String expResult = contents;
String enc = Converter.encryptString(contents, secret);
System.out.println("Encrypted String: " + enc);
String result = Converter.decryptString(enc, secret);
assertEquals(expResult, result);
}
}
deployer/Dockerfile
View file @ 21ce2a62
......@@ -28,4 +28,7 @@ RUN pip3 install --no-cache-dir -r requirements.txt
COPY . /usr/src/app
CMD python3 __main__.py $RABBITMQ_HOST deployer
CMD sed -i "s#http://127.0.0.1:8081/tosca-sure/1.0.0#$SURE_TOSCA_BASE_PATH#g" properties.ini && \
sed -i "s#http://127.0.0.1:3000/api#$SEMAPHORE_BASE_PATH#g" properties.ini && \
sed -i "s#host = 127.0.0.1#host = $RABBITMQ_HOST#g" properties.ini && \
python3 __main__.py $RABBITMQ_HOST deployer
deployer/__main__.py
View file @ 21ce2a62
......@@ -8,14 +8,16 @@ import os
import sys
import tempfile
import time
import traceback
from threading import Thread
from time import sleep
from service.deploy_service import DeployService
from service.tosca_helper import ToscaHelper
import pika
import sure_tosca_client
import yaml
from service.deploy_service import DeployService
from service.tosca_helper import ToscaHelper
logger = logging.getLogger(__name__)
done = False
......@@ -85,9 +87,17 @@ def handle_delivery(message):
nodes_pairs = tosca_helper.get_deployment_node_pairs()
deployService = DeployService(semaphore_base_url=semaphore_base_url, semaphore_username=semaphore_username,
semaphore_password=semaphore_password)
semaphore_password=semaphore_password, vms=tosca_helper.get_vms())
try:
for node_pair in nodes_pairs:
deployService.deploy(node_pair)
updated_node = deployService.deploy(node_pair)
if isinstance(updated_node, list):
for node in updated_node:
tosca_template_dict = tosca_helper.set_node(node,tosca_template_dict)
logger.info("tosca_template_dict :" + json.dumps(tosca_template_dict))
else:
tosca_template_dict = tosca_helper.set_node(updated_node, tosca_template_dict)
logger.info("tosca_template_dict :" + json.dumps(tosca_template_dict))
response = {'toscaTemplate': tosca_template_dict}
output_current_milli_time = int(round(time.time() * 1000))
......@@ -95,6 +105,10 @@ def handle_delivery(message):
logger.info("Returning Deployment")
logger.info("Output message:" + json.dumps(response))
return json.dumps(response)
except Exception as e:
track = traceback.format_exc()
print(track)
raise
def threaded_function(args):
......
deployer/build/lib/test/test_deployer.py
View file @ 21ce2a62
......@@ -39,7 +39,6 @@ class TestDeployer(unittest.TestCase):
sure_tosca_base_url = 'http://127.0.0.1:8081/tosca-sure/1.0.0'
polemarch_base_url='http://127.0.0.1:30001/api/v2'
semaphore_base_url = 'http://127.0.0.1:3000/api'
tosca_service_is_up = ToscaHelper.service_is_up(sure_tosca_base_url)
semaphore_is_up = ToscaHelper.service_is_up(semaphore_base_url)
......
deployer/service/ansible_service.py
View file @ 21ce2a62
......@@ -6,7 +6,7 @@ import datetime
import yaml
from semaphore_client.semaphore_helper import SemaphoreHelper
yaml.Dumper.ignore_aliases = lambda *args : True
yaml.Dumper.ignore_aliases = lambda *args: True
logger = logging.getLogger(__name__)
if not getattr(logger, 'handler_set', None):
......@@ -20,21 +20,20 @@ if not getattr(logger, 'handler_set', None):
class AnsibleService:
def __init__(self, semaphore_base_url=None,semaphore_username=None,semaphore_password=None):
def __init__(self, semaphore_base_url=None, semaphore_username=None, semaphore_password=None):
self.semaphore_base_url = semaphore_base_url
self.semaphore_username = semaphore_username
self.semaphore_password = semaphore_password
self.semaphore_helper = SemaphoreHelper(self.semaphore_base_url, self.semaphore_username, self.semaphore_password)
self.semaphore_helper = SemaphoreHelper(self.semaphore_base_url, self.semaphore_username,
self.semaphore_password)
self.repository_id = None
self.template_id = None
def execute(self,nodes_pair):
vms = nodes_pair[0]
def execute(self, nodes_pair, interface_type, vms, env_vars=None):
application = nodes_pair[1]
name = application.name
desired_state = None
tasks_outputs = {}
interfaces = application.node_template.interfaces
if 'current_state' in application.node_template.attributes:
current_state = application.node_template.attributes['current_state']
......@@ -43,31 +42,45 @@ class AnsibleService:
if desired_state:
now = datetime.datetime.now()
project_id = self.semaphore_helper.create_project(application.name+'_'+str(now))
inventory_contents = yaml.dump( self.build_yml_inventory(vms),default_flow_style=False)
project_id = self.semaphore_helper.create_project(application.name + '_' + str(now))
inventory_contents = yaml.dump(self.build_yml_inventory(vms), default_flow_style=False)
private_key = self.get_private_key(vms)
key_id = self.semaphore_helper.create_ssh_key(application.name, project_id, private_key)
inventory_id = self.semaphore_helper.create_inventory(application.name, project_id, key_id,inventory_contents)
inventory_id = self.semaphore_helper.create_inventory(application.name, project_id, key_id,
inventory_contents)
if 'RUNNING' == desired_state:
standard = interfaces['Standard']
create = standard['create']
interface = interfaces[interface_type]
create = interface['create']
inputs = create['inputs']
git_url = inputs['repository']
playbook_names = inputs['resources']
for playbook_name in playbook_names:
task_id = self.run_task(name, project_id, key_id, git_url, inventory_id, playbook_name)
environment_id = None
if env_vars:
environment_id = self.semaphore_helper.create_environment(project_id, name, env_vars)
task_id = self.run_task(name, project_id, key_id, git_url, inventory_id, playbook_name,
environment_id=environment_id)
if self.semaphore_helper.get_task(project_id, task_id).status != 'success':
break
# logger.info('playbook: ' + playbook_name + ' task_id: ' + str(task_id))
tasks_outputs[task_id] = self.semaphore_helper.get_task_outputs(project_id, task_id)
if self.semaphore_helper.get_task(project_id,task_id).status == 'success':
configure = standard['configure']
if 'configure' in interface and self.semaphore_helper.get_task(project_id, task_id).status == 'success':
configure = interface['configure']
inputs = configure['inputs']
git_url = inputs['repository']
playbook_names = inputs['resources']
for playbook_name in playbook_names:
task_id = self.run_task(name, project_id, key_id, git_url, inventory_id, playbook_name)
environment_id = None
if env_vars:
environment_id = self.semaphore_helper.create_environment(project_id, name, env_vars)
task_id = self.run_task(name, project_id, key_id, git_url, inventory_id, playbook_name,
environment_id=environment_id)
if self.semaphore_helper.get_task(project_id, task_id).status != 'success':
break
# logger.info('playbook: ' + playbook_name + ' task_id: ' + str(task_id))
tasks_outputs[task_id] = self.semaphore_helper.get_task_outputs(project_id, task_id)
return tasks_outputs
def build_yml_inventory(self, vms):
# loader = DataLoader()
......@@ -75,7 +88,7 @@ class AnsibleService:
# variable_manager = VariableManager()
inventory = {}
all = {}
vars = {'ansible_ssh_common_args':'-o StrictHostKeyChecking=no'}
vars = {'ansible_ssh_common_args': '-o StrictHostKeyChecking=no'}
vars['ansible_ssh_user'] = vms[0].node_template.properties['user_name']
children = {}
for vm in vms:
......@@ -100,24 +113,21 @@ class AnsibleService:
private_key = vms[0].node_template.attributes['user_key_pair']['keys']['private_key']
return base64.b64decode(private_key).decode('utf-8').replace(r'\n', '\n')
def run_task(self, name, project_id, key_id, git_url, inventory_id, playbook_name):
logger.info('project_id: '+str(project_id)+ ' task name: ' + str(name)+ ' git url: '+git_url+' playbook: '+playbook_name)
def run_task(self, name, project_id, key_id, git_url, inventory_id, playbook_name, environment_id=None):
logger.info('project_id: ' + str(project_id) + ' task name: ' + str(
name) + ' git url: ' + git_url + ' playbook: ' + playbook_name)
self.repository_id = self.semaphore_helper.create_repository(name, project_id, key_id, git_url)
template_id = self.semaphore_helper.create_template(project_id, key_id, inventory_id, self.repository_id,
playbook_name)
task_id = self.semaphore_helper.execute_task(project_id, template_id, playbook_name)
task_id = self.semaphore_helper.execute_task(project_id, template_id, playbook_name,
environment_id=environment_id)
task = self.semaphore_helper.get_task(project_id, task_id)
last_output = ''
last_status = ''
while task.status == 'waiting' or task.status == 'running':
task = self.semaphore_helper.get_task(project_id, task_id)
logger.info('task name: '+name+ ' task status: ' + str(task.status))
task_outputs = self.semaphore_helper.get_task_outputs(project_id, task_id)
this_output = task_outputs[len(task_outputs)-1].output.replace(r'\n', '\n').replace(r'\r', '\r')
if last_output != this_output:
logger.info('task output: ' + str(this_output))
last_output = this_output
# logger.info('task output: ' + str(latask name:st_output))
this_status = task.status
if last_status != this_status:
logger.info('task name: ' + name + ', task status: ' + str(task.status))
last_status = this_status
sleep(3)
return task_id
deployer/service/deploy_service.py
View file @ 21ce2a62
from service import tosca_helper, ansible_service
import json
import logging
from service import tosca_helper
from service.ansible_service import AnsibleService
logger = logging.getLogger(__name__)
if not getattr(logger, 'handler_set', None):
logger.setLevel(logging.INFO)
h = logging.StreamHandler()
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
h.setFormatter(formatter)
logger.addHandler(h)
logger.handler_set = True
class DeployService:
def __init__(self, polemarch_base_url=None,polemarch_username=None,polemarch_password=None,
semaphore_base_url=None,semaphore_username=None,semaphore_password=None):
semaphore_base_url=None,semaphore_username=None,semaphore_password=None, vms=None):
# self.polemarch_base_url = polemarch_base_url
# self.polemarch_username=polemarch_username
# self.polemarch_password = polemarch_password
self.semaphore_base_url = semaphore_base_url
self.semaphore_username = semaphore_username
self.semaphore_password = semaphore_password
self.vms = vms
for vm in vms:
if vm.node_template.attributes['role'] == 'master':
self.master_ip = vm.node_template.attributes['public_ip']
break
def deploy(self,nodes_pair):
target = nodes_pair[0]
source = nodes_pair[1]
interface_types = tosca_helper.get_interface_types(source)
if interface_types:
ansible_service = AnsibleService(self.semaphore_base_url, self.semaphore_username, self.semaphore_password)
env_vars = self.get_env_vars(nodes_pair)
if 'Standard' in interface_types:
ansible_service = AnsibleService(self.semaphore_base_url,self.semaphore_username,self.semaphore_password)
ansible_service.execute(nodes_pair)
else:
print(interface_types)
task_outputs = ansible_service.execute(nodes_pair, 'Standard', self.vms, env_vars=env_vars)
source = self.set_attributes(task_outputs,source)
if 'Kubernetes' in interface_types:
task_outputs = ansible_service.execute(nodes_pair, 'Kubernetes', self.vms, env_vars=env_vars)
source = self.set_attributes(task_outputs,source)
return source
def get_env_vars(self, nodes_pair):
target = nodes_pair[0]
source = nodes_pair[1]
env_vars = {'K8s_NAMESPACE': 'default'}
if source.node_template.type == 'tosca.nodes.QC.Container.Application.Docker':
env_vars['DOCKER_IMAGE'] = source.node_template.artifacts['image']['file']
env_vars['DOCKER_SERVICE_NAME'] = source.name
env_vars['CONTAINER_PORT'] = source.node_template.properties['ports'][0].split(':')[1]
return env_vars
def set_attributes(self, task_outputs,source):
# target = nodes_pair[0]
# source = nodes_pair[1]
if source.node_template.type == 'tosca.nodes.QC.docker.Orchestrator.Kubernetes':
source = self.set_kubernetes_attributes(source=source,task_outputs=task_outputs)
if source.node_template.type == 'tosca.nodes.QC.Container.Application.Docker':
source = self.set_docker_attributes(source=source, task_outputs=task_outputs)
# lst = list(nodes_pair)
# lst[1] = source
# nodes_pair = tuple(lst)
return source
def parse_ansible_var(self,var_name, output_array):
index = 0
start_index = -1
end_index = -1
for out in output_array:
index += 1
if 'TASK' in out.output or 'PLAY RECAP' in out.output:
if start_index > -1:
end_index = index - 1
break
if start_index <=-1 and '"' + var_name + '":' in out.output:
start_index = index - 1
if start_index <= -1:
return None
ansible_var = output_array[start_index:end_index]
json_ansible_var = '{'
for item in ansible_var:
json_ansible_var = json_ansible_var + item.output
logger.info('found '+var_name +': '+ str(json_ansible_var))
return json.loads(json_ansible_var)
def get_dashboard_token(self, k8s_dashboard_token):
k8s_dashboard_token = k8s_dashboard_token['k8s_dashboard_token']
# if 'resources' in k8s_secrets:
# return self.get_secret_from_k8s_info(k8s_secrets)
if 'stdout' in k8s_dashboard_token:
return self.get_secret_from_stdout(k8s_dashboard_token)
def get_service_port(self, k8s_services, service_name,port_type):
resources = k8s_services['k8s_services']['resources']
for resource in resources:
name = resource['metadata']['name']
if name == service_name:
ports = resource['spec']['ports']
for port in ports:
if port_type in port:
return port[port_type]
return None
def get_secret_from_k8s_info(self, k8s_secrets):
resources = k8s_secrets['resources']
for resource in resources:
metadata = resource['metadata']
if 'admin-user-token' in metadata['name']:
dashboard_token = resource['data']['token']
logger.info('found dashboard_token: ' + str(dashboard_token))
return resource['data']['token']
return None
def get_secret_from_stdout(self, k8s_dashboard_token):
return k8s_dashboard_token['stdout'].replace('token: ', '')
def set_kubernetes_attributes(self, source=None,task_outputs=None):
attributes = source.node_template.attributes
if 'tokens' not in attributes:
tokens = []
attributes['tokens'] = tokens
else:
tokens = attributes['tokens']
if 'dashboard_url' not in source.node_template.attributes:
dashboard_url = ''
attributes['dashboard_url'] = dashboard_url
else:
dashboard_url = attributes['dashboard_url']
k8s_dashboard_token = None
k8s_services = None
for task_output_key in task_outputs:
task_output = task_outputs[task_output_key]
if not k8s_dashboard_token:
k8s_dashboard_token = self.parse_ansible_var('k8s_dashboard_token', task_output)
if not k8s_services:
k8s_services = self.parse_ansible_var('k8s_services', task_output)
if k8s_services and k8s_dashboard_token:
credential = {'token_type': 'k8s_dashboard_token'}
credential['token'] = self.get_dashboard_token(k8s_dashboard_token)
tokens.append(credential)
service_port = self.get_service_port(k8s_services, 'kubernetes-dashboard', 'nodePort')
dashboard_url = 'https://' + self.master_ip + ':' + str(service_port)
attributes['dashboard_url'] = dashboard_url
logger.info('source.node_template.attributes: ' + str(attributes))
return source
def set_docker_attributes(self, source, task_outputs):
attributes = source.node_template.attributes
if 'service_url' not in source.node_template.attributes:
service_url = ''
attributes['service_url'] = service_url
for task_output_key in task_outputs:
task_output = task_outputs[task_output_key]
k8s_services = self.parse_ansible_var('k8s_services', task_output)
service_port = self.get_service_port(k8s_services, source.name, 'nodePort')
if service_port:
service_url = 'http://' + self.master_ip + ':' + str(service_port)
attributes['service_url'] = service_url
logger.info('source.node_template.attributes: ' + str(attributes))
return source
deployer/service/tosca_helper.py
View file @ 21ce2a62
import copy
import os
import sys
import urllib.request
from sure_tosca_client import Configuration, ApiClient
from sure_tosca_client import Configuration, ApiClient, NodeTemplate
from sure_tosca_client.api import default_api
......@@ -28,7 +29,6 @@ class ToscaHelper:
return api
def get_interface_types(target):
print(target.node_template.interfaces)
interface_types = []
for interface in target.node_template.interfaces:
interface_types.append(interface)
......@@ -49,7 +49,7 @@ class ToscaHelper:
# So we explicitly get the VMs
# I don't like this solution but I can't think of something better.
if related_node.node_template.type == 'tosca.nodes.QC.VM.topology':
vms = self.tosca_client.get_node_templates(self.doc_id,type_name='tosca.nodes.QC.VM.Compute')
vms = self.get_vms()
related_node = vms
pair = (related_node, node)
nodes_pairs.append(pair)
......@@ -70,6 +70,16 @@ class ToscaHelper:
return True
def get_vms(self):
return self.tosca_client.get_node_templates(self.doc_id, type_name='tosca.nodes.QC.VM.Compute')
def set_node(self, updated_node, tosca_template_dict):
node_templates = tosca_template_dict['topology_template']['node_templates']
for node_name in node_templates:
if node_name == updated_node.name:
node_templates[node_name] = updated_node.node_template.to_dict()
return tosca_template_dict
def get_interface_types(node):
interface_type_names = []
......
deployer/test/test_deployer.py
View file @ 21ce2a62
......@@ -39,7 +39,6 @@ class TestDeployer(unittest.TestCase):
sure_tosca_base_url = 'http://127.0.0.1:8081/tosca-sure/1.0.0'
polemarch_base_url='http://127.0.0.1:30001/api/v2'
semaphore_base_url = 'http://127.0.0.1:3000/api'
tosca_service_is_up = ToscaHelper.service_is_up(sure_tosca_base_url)
semaphore_is_up = ToscaHelper.service_is_up(semaphore_base_url)
......@@ -53,8 +52,9 @@ class TestDeployer(unittest.TestCase):
self.assertIsNotNone(nodes_pairs)
username = 'admin'
deployService = DeployService(polemarch_base_url=polemarch_base_url,polemarch_username=username,polemarch_password='admin',
semaphore_base_url=semaphore_base_url,semaphore_username=username,semaphore_password='password')
deployService = DeployService(polemarch_username=username,polemarch_password='admin',
semaphore_base_url=semaphore_base_url,semaphore_username=username,
semaphore_password='password')
for node_pair in nodes_pairs:
deployService.deploy(node_pair)
......
docker-compose/docker-compose.yml
View file @ 21ce2a62
version: '3'
services:
#nginx:
#image: nginx
#volumes:
#- ./nginx.conf:/etc/nginx/nginx.conf
#- ./cert.pem:/etc/nginx/cert.pem
#- ./privkey.pem:/etc/nginx/privkey.pem
##- ./www:/data/www
#ports:
#- "80:80"
#- "443:443"
nginx:
image: nginx
volumes:
- ./nginx.conf:/etc/nginx/nginx.conf
- ./cert.pem:/etc/nginx/cert.pem
- ./privkey.pem:/etc/nginx/privkey.pem
#- ./www:/data/www
ports:
- "80:80"
- "443:443"
rabbit:
image: rabbitmq:3.8-management
......@@ -55,21 +55,22 @@ services:
- "27017:27017"
#manager:
#depends_on:
#- rabbit
#- mongo
#- sure-tosca
#image: qcdis/manager:3.0.0
#environment:
#RABBITMQ_HOST: rabbit
#MONGO_HOST: mongo
#SURE_TOSCA_BASE_PATH: http://sure-tosca:8081/tosca-sure/1.0.0
#ports:
#- "8080:8080"
manager:
depends_on:
- rabbit
- mongo
- sure-tosca
image: qcdis/manager
environment:
RABBITMQ_HOST: rabbit
MONGO_HOST: mongo
SURE_TOSCA_BASE_PATH: http://sure-tosca:8081/tosca-sure/1.0.0
CREDENTIAL_SECRET: 123
ports:
- "8080:8080"
sure-tosca:
image: qcdis/sure-tosca:3.0.0
image: qcdis/sure-tosca
ports:
- "8081:8081"
......@@ -77,27 +78,32 @@ services:
depends_on:
- rabbit
- sure-tosca
image: qcdis/planner:3.0.0
image: qcdis/planner
environment:
RABBITMQ_HOST: rabbit
provisioner:
depends_on:
- rabbit
- sure-tosca
image: qcdis/provisioner
environment:
RABBITMQ_HOST: rabbit
SURE_TOSCA_BASE_PATH: http://sure-tosca:8081/tosca-sure/1.0.0
CLOUD_STORM_SECRET: 456
CREDENTIAL_SECRET: 123
deployer:
depends_on:
- rabbit
- sure-tosca
image: qcdis/deployer
environment:
RABBITMQ_HOST: rabbit
SURE_TOSCA_BASE_PATH: http://sure-tosca:8081/tosca-sure/1.0.0
SEMAPHORE_BASE_PATH: http://semaphore:3000/api
#provisioner:
#depends_on:
#- rabbit
#- sure-tosca
#image: qcdis/provisioner:3.0.0
#environment:
#RABBITMQ_HOST: rabbit
#SURE_TOSCA_BASE_PATH: http://sure-tosca:8081/tosca-sure/1.0.0
#deployer:
#depends_on:
#- rabbit
#- sure-tosca
#image: qcdis/deployer:3.0.0
#environment:
#RABBITMQ_HOST: rabbit
#SURE_TOSCA_BASE_PATH: http://sure-tosca:8081/tosca-sure/1.0.0
docker-compose/nginx.conf
View file @ 21ce2a62
worker_processes auto;
events {
worker_connections 1024;
use epoll;
......@@ -7,19 +8,17 @@ events {
}
http {
proxy_connect_timeout 9200;
proxy_send_timeout 9200;
proxy_read_timeout 9200;
send_timeout 9200;
tcp_nodelay on;
# this is necessary for us to be able to disable request buffering in all cases
proxy_http_version 1.1;
# upstream semaphore {
# server semaphore:3000;
# }
# upstream manager {
# server manager:8080;
# }
server {
listen [::]:80 default_server;
server_name _;
......@@ -61,16 +60,9 @@ http {
proxy_request_buffering off;
}
# location /api/ws {
# proxy_pass http://semaphore:3000/api/ws;
# proxy_http_version 1.1;
# proxy_set_header Upgrade $http_upgrade;
# proxy_set_header Connection "upgrade";
# proxy_set_header Origin "";
# }
location /manger/ {
proxy_pass http://manager:8080/manger/;
location /manager {
proxy_pass http://manager:8080/manager;
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
......@@ -81,8 +73,8 @@ http {
proxy_request_buffering off;
}
location /manger/swagger-ui.html {
proxy_pass http://manager:8080/swagger-ui.html/;
location /swagger-ui.html {
proxy_pass http://manager:8080/swagger-ui.html;
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
......@@ -93,8 +85,8 @@ http {
proxy_request_buffering off;
}
location /tosca-sure/1.0.0/ {
proxy_pass http://sure-tosca:8081/tosca-sure/1.0.0/;
location /tosca-sure/1.0.0 {
proxy_pass http://sure-tosca:8081/tosca-sure/1.0.0;
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
......
example_messages/message_provision_request.json
View file @ 21ce2a62
{
"owner": "user",
"creationDate": 1585660972026,
"creationDate": 1591357712632,
"toscaTemplate": {
"tosca_definitions_version": "tosca_simple_yaml_1_0",
"tosca_default_namespace": null,
......@@ -11,7 +11,7 @@
"node_templates": {
"compute": {
"properties": {
"disk_size": "10000 MB",
"disk_size": "20000 MB",
"mem_size": "1000 MB",
"num_cores": 1,
"os": "Ubuntu 18.04",
......@@ -26,7 +26,7 @@
},
"compute_1": {
"properties": {
"disk_size": "10000 MB",
"disk_size": "20000 MB",
"mem_size": "1000 MB",
"num_cores": 1,
"os": "Ubuntu 18.04",
......@@ -39,7 +39,11 @@
},
"type": "tosca.nodes.QC.VM.Compute"
},
"kubernetes": {
"glusterfs": {
"properties": {
"min_masters_num": 1,
"min_workers_num": 1
},
"requirements": [
{
"host": {
......@@ -53,25 +57,60 @@
"Standard": {
"configure": {
"inputs": {
"playbook": "k8s_dashboard.yaml",
"repository": "https://github.com/skoulouzis/playbooks.git"
"repository": "https://github.com/QCDIS/mysome_glusterfs.git",
"resources": [
"002.setup_glusterfs_infra.yml",
"003.setup_glusterfs_cluster.yml",
"004.mount_glusterfs.yml"
]
}
},
"create": {
"inputs": {
"playbook": "install_k8s.yaml",
"repository": "https://github.com/skoulouzis/playbooks.git"
"repository": "https://github.com/QCDIS/playbooks.git",
"resources": [
"ec2/ec2_mount.yaml",
"install_glusterfs_prerequisites.yaml"
]
}
}
}
},
"delete": {
"type": "tosca.nodes.QC.Application.GlusterFS"
},
"tic": {
"requirements": [
{
"storage": {
"capability": "tosca.capabilities.QC.GlusterFS",
"node": "glusterfs",
"relationship": "tosca.relationships.DependsOn"
}
}
],
"interfaces": {
"Standard": {
"create": {
"inputs": {
"playbook": "k8s_delete.yaml",
"repository": "https://github.com/skoulouzis/playbooks.git"
"repository": "https://github.com/bityoga/fabric_as_code.git",
"resources": [
"011.initialize_hosts.yml",
"012.prepare_docker_images.yml",
"013.mount_fs.yml",
"014.spawn_swarm.yml",
"015.deploy_swarm_visualizer.yml",
"016.deploy_portainer.yml",
"100.deploy_ca.yml",
"101.deploy_orderer.yml",
"102.deploy_peers.yml",
"103.deploy_cli.yml",
"104.deploy_hlf_explorer.yml"
]
}
}
}
},
"type": "tosca.nodes.QC.docker.Orchestrator.Kubernetes"
"type": "tosca.nodes.QC.Application.TIC"
},
"topology": {
"properties": {
......@@ -133,9 +172,9 @@
"credential": {
"cloud_provider_name": "EC2",
"keys": {
"aws_access_key_id": "XXXXXXXXXXXX"
"aws_access_key_id": "BGCAc9htRRjcZx5uZ6aWnRo67tNAgk7M26DwuN3cZvypnCt6V56U9RbEuRpmXbER"
},
"token": "XXXXXXXXXXXXX",
"token": "BGCAc9htRRjcZx5uZ6aWnRo67tNAgk7M26DwuN3cZvypnCt6V56U9RbEuRpmXbER",
"token_type": "access_key"
},
"desired_state": "RUNNING"
......@@ -146,30 +185,6 @@
"type": "string"
}
}
},
"ws-pema": {
"properties": {
"ports": [
"30001:8080"
]
},
"requirements": [
{
"host": {
"capability": "tosca.capabilities.QC.docker.Orchestrator",
"node": "kubernetes",
"relationship": "tosca.relationships.HostedOn"
}
}
],
"type": "tosca.nodes.QC.Container.Application.Docker",
"artifacts": {
"image": {
"file": "alogo53/ws-pema-lifewatch",
"repository": "docker_hub",
"type": "tosca.artifacts.Deployment.Image.Container.Docker"
}
}
}
},
"relationship_templates": null,
......@@ -183,19 +198,19 @@
"description": "TOSCA example",
"imports": [
{
"nodes": "https://raw.githubusercontent.com/skoulouzis/DRIP/develop/TOSCA/types/nodes.yaml"
"nodes": "https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/types/nodes.yaml"
},
{
"data": "https://raw.githubusercontent.com/skoulouzis/CONF/develop/TOSCA/types/data.yml"
"data": "https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/types/data.yml"
},
{
"capabilities": "https://raw.githubusercontent.com/skoulouzis/DRIP/develop/TOSCA/types/capabilities.yaml"
"capabilities": "https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/types/capabilities.yaml"
},
{
"policies": "https://raw.githubusercontent.com/skoulouzis/DRIP/develop/TOSCA/types/policies.yaml"
"policies": "https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/types/policies.yaml"
},
{
"interfaces": "https://raw.githubusercontent.com/skoulouzis/DRIP/develop/TOSCA/types/interfaces.yml"
"interfaces": "https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/types/interfaces.yml"
}
],
"dsl_definitions": null,
......
k8s/CONF/deployer-deployment.yaml
View file @ 21ce2a62
......@@ -33,7 +33,9 @@ spec:
value: rabbit
- name: SURE_TOSCA_BASE_PATH
value: http://sure-tosca:8081/tosca-sure/1.0.0
image: qcdis/deployer:3.0.0
- name: SEMAPHORE_BASE_PATH
value: http://semaphore:3000/api
image: qcdis/deployer
name: deployer
imagePullPolicy: Always
resources: {}
......
k8s/CONF/manager-deployment.yaml
View file @ 21ce2a62
......@@ -35,6 +35,8 @@ spec:
value: rabbit
- name: SURE_TOSCA_BASE_PATH
value: http://sure-tosca:8081/tosca-sure/1.0.0
- name: CREDENTIAL_SECRET
value: top_secret
image: qcdis/manager:3.0.0
name: manager
imagePullPolicy: Always
......
k8s/CONF/manager-deployment.yaml.template 0 → 100644
View file @ 21ce2a62
apiVersion: apps/v1
kind: Deployment
metadata:
namespace: conf
annotations:
kompose.cmd: kompose convert
kompose.version: 1.16.0 (0c01309)
creationTimestamp: null
labels:
io.kompose.service: manager
name: manager
spec:
selector:
matchLabels:
io.kompose.service: manager
replicas: 1
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 50%
maxUnavailable: 50%
minReadySeconds: 10
revisionHistoryLimit: 3
template:
metadata:
creationTimestamp: null
labels:
io.kompose.service: manager
spec:
containers:
- env:
- name: MONGO_HOST
value: mongo
- name: RABBITMQ_HOST
value: rabbit
- name: SURE_TOSCA_BASE_PATH
value: http://sure-tosca:8081/tosca-sure/1.0.0
- name: CREDENTIAL_SECRET
value: top_secret
image: qcdis/manager:3.0.0
name: manager
imagePullPolicy: Always
ports:
- containerPort: 8080
resources: {}
restartPolicy: Always
status: {}
k8s/CONF/nginx-service.yaml
View file @ 21ce2a62
......@@ -8,10 +8,6 @@ metadata:
spec:
type: NodePort
ports:
- port: 80
nodePort: 30000
protocol: TCP
name: http
- port: 443
nodePort: 30001
protocol: TCP
......
k8s/CONF/planner-deployment.yaml
View file @ 21ce2a62
......@@ -31,7 +31,7 @@ spec:
- env:
- name: RABBITMQ_HOST
value: rabbit
image: qcdis/planner:3.0.0
image: qcdis/planner
name: planner
imagePullPolicy: Always
resources: {}
......
k8s/CONF/provisioner-deployment.yaml.template 0 → 100644
View file @ 21ce2a62
apiVersion: apps/v1
kind: Deployment
metadata:
namespace: conf
annotations:
kompose.cmd: kompose convert
kompose.version: 1.16.0 (0c01309)
creationTimestamp: null
labels:
io.kompose.service: provisioner
name: provisioner
spec:
selector:
matchLabels:
io.kompose.service: provisioner
replicas: 1
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 50%
maxUnavailable: 50%
minReadySeconds: 10
revisionHistoryLimit: 3
template:
metadata:
creationTimestamp: null
labels:
io.kompose.service: provisioner
spec:
containers:
- env:
- name: RABBITMQ_HOST
value: rabbit
- name: SURE_TOSCA_BASE_PATH
value: http://sure-tosca:8081/tosca-sure/1.0.0
image: qcdis/provisioner:3.0.0
name: provisioner
imagePullPolicy: Always
resources: {}
restartPolicy: Always
status: {}
k8s/CONF/sure-tosca-deployment.yaml
View file @ 21ce2a62
......@@ -28,7 +28,7 @@ spec:
io.kompose.service: sure-tosca
spec:
containers:
- image: qcdis/sure-tosca:3.0.0
- image: qcdis/sure-tosca
name: sure-tosca
imagePullPolicy: Always
ports:
......
manager/Dockerfile
View file @ 21ce2a62
......@@ -5,6 +5,7 @@ CMD jar -xf manager-3.0.0.jar BOOT-INF/classes/application.properties && \
sed -ie "s#^message.broker.host=.*#message.broker.host=$RABBITMQ_HOST#" BOOT-INF/classes/application.properties && \
sed -ie "s#^db.host=.*#db.host=$MONGO_HOST#" BOOT-INF/classes/application.properties && \
sed -ie "s#^sure-tosca.base.path=.*#sure-tosca.base.path=$SURE_TOSCA_BASE_PATH#" BOOT-INF/classes/application.properties && \
sed -ie "s#^credential.secret=.*#credential.secret=$CREDENTIAL_SECRET#" BOOT-INF/classes/application.properties && \
cat BOOT-INF/classes/application.properties && \
jar -uf manager-3.0.0.jar BOOT-INF/classes/application.properties && \
java -jar manager-3.0.0.jar
manager/src/main/java/nl/uva/sne/drip/api/CredentialApiController.java
View file @ 21ce2a62
......@@ -3,6 +3,9 @@ package nl.uva.sne.drip.api;
import nl.uva.sne.drip.model.tosca.Credential;
import com.fasterxml.jackson.databind.ObjectMapper;
import io.swagger.annotations.*;
import java.io.UnsupportedEncodingException;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.http.HttpStatus;
......@@ -12,6 +15,10 @@ import org.springframework.web.bind.annotation.RequestBody;
import javax.validation.Valid;
import javax.servlet.http.HttpServletRequest;
import java.util.List;
import java.util.logging.Level;
import javax.crypto.BadPaddingException;
import javax.crypto.IllegalBlockSizeException;
import javax.crypto.NoSuchPaddingException;
import nl.uva.sne.drip.service.CredentialService;
import org.springframework.beans.factory.annotation.Autowired;
......@@ -41,8 +48,12 @@ public class CredentialApiController implements CredentialApi {
@Valid @RequestBody Credential body) {
String accept = request.getHeader("Accept");
if (accept != null && accept.contains("application/json")) {
try {
String id = credentialService.save(body);
return new ResponseEntity<>(id, HttpStatus.OK);
} catch (UnsupportedEncodingException | NoSuchAlgorithmException | NoSuchPaddingException | InvalidKeyException | IllegalBlockSizeException | BadPaddingException ex) {
return new ResponseEntity<>(HttpStatus.INTERNAL_SERVER_ERROR);
}
} else {
return new ResponseEntity<>(HttpStatus.NOT_ACCEPTABLE);
}
......
manager/src/main/java/nl/uva/sne/drip/api/ToscaTemplateApiController.java
View file @ 21ce2a62
......@@ -54,7 +54,7 @@ public class ToscaTemplateApiController implements ToscaTemplateApi {
if (accept != null && accept.contains("text/plain")) {
try {
dripService.delete(id, nodeName);
return new ResponseEntity<>("", HttpStatus.OK);
return new ResponseEntity<>(id, HttpStatus.OK);
} catch (IOException | ApiException | TypeExeption | TimeoutException | InterruptedException ex) {
java.util.logging.Logger.getLogger(ToscaTemplateApiController.class.getName()).log(Level.SEVERE, null, ex);
return new ResponseEntity<>(HttpStatus.INTERNAL_SERVER_ERROR);
......
manager/src/main/java/nl/uva/sne/drip/service/CredentialService.java
View file @ 21ce2a62
......@@ -5,13 +5,22 @@
*/
package nl.uva.sne.drip.service;
import com.fasterxml.jackson.core.JsonProcessingException;
import java.io.UnsupportedEncodingException;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import javax.crypto.BadPaddingException;
import javax.crypto.IllegalBlockSizeException;
import javax.crypto.NoSuchPaddingException;
import nl.uva.sne.drip.commons.utils.Converter;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import nl.uva.sne.drip.dao.CredentialDAO;
import nl.uva.sne.drip.model.tosca.Credential;
import org.springframework.beans.factory.annotation.Value;
/**
*
......@@ -20,17 +29,21 @@ import nl.uva.sne.drip.model.tosca.Credential;
@Service
public class CredentialService {
@Value("${credential.secret}")
private String credentialSecret;
@Autowired
private CredentialDAO dao;
public String save(Credential document) {
dao.save(document);
public String save(Credential document) throws UnsupportedEncodingException, NoSuchAlgorithmException, NoSuchPaddingException, InvalidKeyException, IllegalBlockSizeException, BadPaddingException {
dao.save(Converter.encryptCredential(document,credentialSecret));
return document.getId();
}
public Credential findByID(String id) throws JsonProcessingException {
Credential credentials = dao.findById(id).get();
return credentials;
public Credential findByID(String id) {
Credential credential = dao.findById(id).get();
return credential;
}
public void deleteByID(String id) {
......@@ -54,4 +67,6 @@ public class CredentialService {
return dao.findBycloudProviderName(provider);
}
}
manager/src/main/java/nl/uva/sne/drip/service/DRIPService.java
View file @ 21ce2a62
......@@ -67,8 +67,8 @@ public class DRIPService {
message.setToscaTemplate(toscaTemplate);
caller.setRequestQeueName(requestQeueName);
Message plannerResponse = caller.call(message);
ToscaTemplate updatedToscaTemplate = plannerResponse.getToscaTemplate();
Message response = caller.call(message);
ToscaTemplate updatedToscaTemplate = response.getToscaTemplate();
caller.close();
return toscaTemplateService.save(updatedToscaTemplate);
......
manager/src/main/resources/application.properties
View file @ 21ce2a62
......@@ -19,3 +19,5 @@ db.username=drip-user
db.password=drip-pass
sure-tosca.base.path=http://localhost:8081/tosca-sure/1.0.0
credential.secret=top_secret
\ No newline at end of file
manager/src/test/java/nl/uva/sne/drip/service/ServiceTests.java
View file @ 21ce2a62
......@@ -28,6 +28,8 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.util.HashMap;
import java.util.List;
......@@ -37,6 +39,9 @@ import java.util.Set;
import java.util.concurrent.TimeoutException;
import java.util.logging.Level;
import java.util.logging.Logger;
import javax.crypto.BadPaddingException;
import javax.crypto.IllegalBlockSizeException;
import javax.crypto.NoSuchPaddingException;
import nl.uva.sne.drip.Swagger2SpringBoot;
import nl.uva.sne.drip.api.NotFoundException;
import nl.uva.sne.drip.commons.utils.Constants;
......@@ -301,7 +306,7 @@ public class ServiceTests {
* Test of save method, of class CredentialService.
*/
@Test
public void testCredentialServiceSave() {
public void testCredentialServiceSave() throws UnsupportedEncodingException, NoSuchAlgorithmException, NoSuchPaddingException, InvalidKeyException, IllegalBlockSizeException, BadPaddingException {
Logger.getLogger(ServiceTests.class.getName()).log(Level.INFO, "save");
saveCredential();
}
......@@ -332,7 +337,7 @@ public class ServiceTests {
assertEquals(keystorFileChecksum, keystorFileCopyChecksum);
}
public String saveCredential() {
public String saveCredential() throws UnsupportedEncodingException, NoSuchAlgorithmException, NoSuchPaddingException, InvalidKeyException, IllegalBlockSizeException, BadPaddingException {
Logger.getLogger(ServiceTests.class.getName()).log(Level.INFO, "saveCredential");
Credential document = new Credential();
document.setCloudProviderName("exogeni");
......@@ -364,7 +369,7 @@ public class ServiceTests {
* @throws com.fasterxml.jackson.core.JsonProcessingException
*/
@Test
public void testCredentialServiceDeleteByID() throws JsonProcessingException {
public void testCredentialServiceDeleteByID() throws JsonProcessingException, UnsupportedEncodingException, NoSuchAlgorithmException, NoSuchPaddingException, InvalidKeyException, IllegalBlockSizeException, BadPaddingException {
Logger.getLogger(ServiceTests.class.getName()).log(Level.INFO, "deleteByID");
String id = saveCredential();
credentialService.deleteByID(id);
......@@ -382,7 +387,7 @@ public class ServiceTests {
* Test of getAllIds method, of class CredentialService.
*/
@Test
public void testCredentialServiceGetAllIds() {
public void testCredentialServiceGetAllIds() throws UnsupportedEncodingException, NoSuchAlgorithmException, NoSuchPaddingException, InvalidKeyException, IllegalBlockSizeException, BadPaddingException {
Logger.getLogger(ServiceTests.class.getName()).log(Level.INFO, "getAllIds");
testCredentialServiceDeleteAll();
int numOfINst = 3;
......@@ -404,7 +409,7 @@ public class ServiceTests {
}
@Test
public void testSetProvisionerOperation() throws FileNotFoundException, IOException, MissingCredentialsException, ApiException, TypeExeption, JsonProcessingException, TimeoutException, InterruptedException, NotFoundException, MissingVMTopologyException {
public void testSetProvisionerOperation() throws FileNotFoundException, IOException, MissingCredentialsException, ApiException, TypeExeption, JsonProcessingException, TimeoutException, InterruptedException, NotFoundException, MissingVMTopologyException, UnsupportedEncodingException, NoSuchAlgorithmException, NoSuchPaddingException, InvalidKeyException, IllegalBlockSizeException, BadPaddingException {
if (ToscaHelper.isServiceUp(sureToscaBasePath) && ToscaHelper.isServiceUp("http://" + messageBrokerHost + ":15672")) {
addRandomCredential("ExoGENI");
......@@ -447,7 +452,7 @@ public class ServiceTests {
}
}
private void addRandomCredential(String providerName) {
private void addRandomCredential(String providerName) throws UnsupportedEncodingException, NoSuchAlgorithmException, NoSuchPaddingException, InvalidKeyException, IllegalBlockSizeException, BadPaddingException {
Credential document = new Credential();
document.setCloudProviderName(providerName);
Map<String, String> keys = new HashMap<>();
......
provisioner/Dockerfile
View file @ 21ce2a62
......@@ -6,8 +6,14 @@ CMD jar -xf provisioner-3.0.0-jar-with-dependencies.jar application.properties &
cat application.properties && \
sed -ie "s#^message.broker.host=.*#message.broker.host=$RABBITMQ_HOST#" application.properties && \
sed -ie "s#^sure-tosca.base.path=.*#sure-tosca.base.path=$SURE_TOSCA_BASE_PATH#" application.properties && \
sed -ie "s#^cloud.storm.secret=.*#cloud.storm.secret=$CLOUD_STORM_SECRET#" application.properties && \
sed -ie "s#^credential.secret=.*#credential.secret=$CREDENTIAL_SECRET#" application.properties && \
sed -ie "s#^cloud.storm.db.path=.*#cloud.storm.db.path=/etc/UD#" application.properties && \
echo "" >> application.properties && \
echo "cloud.storm.db.path=/etc/UD" >> application.properties && \
cat application.properties && \
jar -uf provisioner-3.0.0-jar-with-dependencies.jar application.properties && \
sleep 5 && \
java -jar provisioner-3.0.0-jar-with-dependencies.jar
provisioner/src/main/java/nl/uva/sne/drip/provisioner/CloudStormService.java
View file @ 21ce2a62
......@@ -17,9 +17,12 @@ import com.jcraft.jsch.KeyPair;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.nio.file.attribute.PosixFilePermission;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
......@@ -29,6 +32,9 @@ import java.util.Properties;
import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;
import javax.crypto.BadPaddingException;
import javax.crypto.IllegalBlockSizeException;
import javax.crypto.NoSuchPaddingException;
import nl.uva.sne.drip.commons.utils.Constants;
import static nl.uva.sne.drip.commons.utils.Constants.*;
import nl.uva.sne.drip.commons.utils.Converter;
......@@ -62,6 +68,9 @@ import topology.analysis.TopologyAnalysisMain;
*/
class CloudStormService {
protected String secret;
private String credentialSecret;
/**
* @return the helper
*/
......@@ -105,6 +114,15 @@ class CloudStormService {
if (sureToscaBasePath == null) {
throw new NullPointerException("sureToscaBasePath cannot be null");
}
secret = properties.getProperty("cloud.storm.secret");
if (secret == null) {
throw new NullPointerException("secret cannot be null");
}
credentialSecret = properties.getProperty("credential.secret");
if (credentialSecret == null) {
throw new NullPointerException("secret cannot be null");
}
Logger.getLogger(CloudStormService.class.getName()).log(Level.FINE, "sureToscaBasePath: {0}", sureToscaBasePath);
this.helper = new ToscaHelper(sureToscaBasePath);
this.helper.uploadToscaTemplate(toscaTemplate);
......@@ -123,9 +141,10 @@ class CloudStormService {
for (NodeTemplateMap vmTopologyMap : helper.getVMTopologyTemplates()) {
Map<String, Object> provisionedFiles = helper.getNodeArtifact(vmTopologyMap.getNodeTemplate(), "provisioned_files");
if (provisionedFiles != null) {
String fileContentsBase64 = (String) provisionedFiles.get("file_contents");
if (fileContentsBase64 != null) {
String encryptedFileContents = (String) provisionedFiles.get("file_contents");
if (encryptedFileContents != null) {
File zipFile = new File(tempInputDir.getParent() + File.separator + Long.toString(System.nanoTime()) + "-" + CLOUD_STORM_FILES_ZIP_SUFIX);
String fileContentsBase64 = Converter.decryptString(encryptedFileContents, secret);
Converter.decodeBase64BToFile(fileContentsBase64, zipFile.getAbsolutePath());
Converter.unzipFolder(zipFile.getAbsolutePath(), tempInputDir.getAbsolutePath());
......@@ -282,6 +301,7 @@ class CloudStormService {
int i = 0;
for (NodeTemplateMap vmTopologyMap : vmTopologiesMaps) {
Credential toscaCredentials = getHelper().getCredentialsFromVMTopology(vmTopologyMap);
toscaCredentials = Converter.dencryptCredential(toscaCredentials, credentialSecret);
CloudCred cloudStormCredential = new CloudCred();
cloudStormCredential.setCloudProvider(toscaCredentials.getCloudProviderName());
String credInfoFile = toscaCredentials.getCloudProviderName() + i + ".yml";
......@@ -355,7 +375,7 @@ class CloudStormService {
FileUtils.copyDirectory(srcDir, destDir);
}
protected ToscaTemplate runCloudStorm(String tempInputDirPath, boolean dryRun) throws IOException, ApiException {
protected ToscaTemplate runCloudStorm(String tempInputDirPath, boolean dryRun) throws IOException, ApiException, UnsupportedEncodingException, NoSuchAlgorithmException, NoSuchPaddingException, InvalidKeyException, IllegalBlockSizeException, BadPaddingException {
String[] args = new String[]{"run", tempInputDirPath};
File topTopologyFile = new File(tempInputDirPath + TOPOLOGY_RELATIVE_PATH
+ TOP_TOPOLOGY_FILE_NAME);
......@@ -402,7 +422,7 @@ class CloudStormService {
return keyPair;
}
protected NodeTemplateMap addCloudStromArtifacts(NodeTemplateMap vmTopologyMap, String tempInputDirPath) throws IOException {
protected NodeTemplateMap addCloudStromArtifacts(NodeTemplateMap vmTopologyMap, String tempInputDirPath) throws IOException, UnsupportedEncodingException, NoSuchAlgorithmException, NoSuchPaddingException, InvalidKeyException, IllegalBlockSizeException, BadPaddingException {
Map<String, Object> artifacts = vmTopologyMap.getNodeTemplate().getArtifacts();
if (artifacts == null) {
artifacts = new HashMap<>();
......@@ -416,7 +436,8 @@ class CloudStormService {
Logger.getLogger(CloudStormService.class.getName()).log(Level.FINE, "Created zip at: {0}", zipPath);
String cloudStormZipFileContentsAsBase64 = Converter.encodeFileToBase64Binary(zipPath);
provisionedFiles.put("file_contents", cloudStormZipFileContentsAsBase64);
String encryptedCloudStormZipFileContents = Converter.encryptString(cloudStormZipFileContentsAsBase64, secret);
provisionedFiles.put("file_contents", encryptedCloudStormZipFileContents);
provisionedFiles.put("encoding", "base64");
provisionedFiles.put("file_ext", "zip");
artifacts.put("provisioned_files", provisionedFiles);
......@@ -425,7 +446,7 @@ class CloudStormService {
return vmTopologyMap;
}
private void setSSHKeysToVMAttributes(int i, List<NodeTemplateMap> vmTopologiesMaps, CloudsStormSubTopology subTopology, String tempInputDirPath) throws IOException, ApiException {
private void setSSHKeysToVMAttributes(int i, List<NodeTemplateMap> vmTopologiesMaps, CloudsStormSubTopology subTopology, String tempInputDirPath) throws IOException, ApiException, UnsupportedEncodingException, NoSuchAlgorithmException, NoSuchPaddingException, InvalidKeyException, IllegalBlockSizeException, BadPaddingException {
NodeTemplateMap vmTopologyMap = vmTopologiesMaps.get(i);
vmTopologyMap = addCloudStromArtifacts(vmTopologyMap, tempInputDirPath);
......@@ -459,7 +480,7 @@ class CloudStormService {
userKyes.put("private_key", Converter.encodeFileToBase64Binary(userKyePairFolder + File.separator + "id_rsa"));
userKyes.put("public_key", Converter.encodeFileToBase64Binary(userKyePairFolder + File.separator + "id_rsa.pub"));
userKeyPairCredential.setKeys(userKyes);
// userKeyPairCredential = Converter.encryptCredential(userKeyPairCredential, credentialSecret);
CloudsStormVMs cloudsStormVMs = objectMapper.readValue(new File(tempInputDirPath + TOPOLOGY_RELATIVE_PATH + File.separator + subTopology.getTopology() + ".yml"),
CloudsStormVMs.class);
List<CloudsStormVM> vms = cloudsStormVMs.getVms();
......
provisioner/src/main/resources/application.properties
View file @ 21ce2a62
......@@ -5,4 +5,6 @@ message.broker.queue.provisioner=provisioner
message.broker.queue.planner=planner
message.broker.queue.deployer=deployer
sure-tosca.base.path=http://localhost:8081/tosca-sure/1.0.0
cloud.storm.secret=top_secret
credential.secret=top_secret
cloud.storm.db.path=etc/UD
\ No newline at end of file
provisioner/src/test/java/nl/uva/sne/drip/provisioner/CloudStormServiceTest.java
View file @ 21ce2a62
......@@ -284,6 +284,7 @@ public class CloudStormServiceTest {
assertTrue(provisionedFiles.containsKey("file_contents"));
String cloudStormZipFileContentsAsBase64 = provisionedFiles.get("file_contents");
cloudStormZipFileContentsAsBase64 = Converter.decryptString(cloudStormZipFileContentsAsBase64, instance.secret);
String testZipPath = tempInputDirPath + File.separator + "test.zip";
Converter.decodeBase64BToFile(cloudStormZipFileContentsAsBase64, testZipPath);
......
provisioner/src/test/resources/application.properies deleted 100644 → 0
View file @ 0202be19
message.broker.host=127.0.0.1
message.broker.username=guest
message.broker.password=guest
message.broker.queue.provisioner=provisioner
message.broker.queue.planner=planner
message.broker.queue.deployer=deployer
sure-tosca.base.path=http://localhost:8081/tosca-sure/1.0.0
semaphore-python-client-generated/semaphore_client/semaphore_helper.py
View file @ 21ce2a62
import json
from datetime import datetime
import urllib3
from semaphore_client import Configuration, ApiClient, api, ProjectRequest, Login, AccessKeyRequest, InventoryRequest, \
RepositoryRequest, TemplateRequest, Task
RepositoryRequest, TemplateRequest, Task, EnvironmentRequest
class SemaphoreHelper:
......@@ -77,8 +78,12 @@ class SemaphoreHelper:
templates = self.project_api.project_project_id_templates_get(project_id, playbook_name, 'asc')
return self.find_template(templates,playbook_name).id
def execute_task(self, project_id, template_id, playbook_name):
task = Task(template_id=template_id, playbook=playbook_name)
def execute_task(self, project_id, template_id, playbook_name,environment_id=None):
environment_json = None
if environment_id:
environment = self.find_environment(project_id,environment_id=environment_id)
environment_json = environment.json
task = Task(template_id=template_id, playbook=playbook_name,environment=environment_json)
self.project_api.project_project_id_tasks_post(task,project_id)
tasks = self.project_api.project_project_id_tasks_get(project_id)
return self.find_latest_task(tasks).id
......@@ -127,3 +132,20 @@ class SemaphoreHelper:
if repo.git_url == git_url:
return repo
return None
def create_environment(self, project_id,name,env_vars):
vars = {"ENV":env_vars}
vars_str = json.dumps(vars)
environment_request = EnvironmentRequest(name=name, project_id=project_id, password=None, json=vars_str)
self.project_api.project_project_id_environment_post(environment_request,project_id)
environment = self.find_environment(project_id, name,vars_str)
return environment.id
def find_environment(self, project_id, name=None,vars_str=None,environment_id=None):
environments = self.project_api.project_project_id_environment_get(project_id,'db-deploy','desc')
for environment in environments:
if environment_id and environment.id ==environment_id:
return environment
if environment.name and environment.name == name and environment.project_id == project_id and environment.json == vars_str:
return environment
return None
\ No newline at end of file
semaphore-python-client-generated/test/test_task.py
View file @ 21ce2a62
......@@ -15,7 +15,7 @@ from __future__ import absolute_import
import unittest
from time import sleep
import datetime
import yaml
from semaphore_client.semaphore_helper import SemaphoreHelper
......@@ -30,22 +30,23 @@ class TestTask(unittest.TestCase):
if SemaphoreHelper.service_is_up(self.semaphore_base_url):
self.username = 'admin'
self.password = 'password'
self.project_name = 'test'
now = datetime.datetime.now()
self.project_name = 'project_name_test'+'_'+str(now)
self.private_key = '-----BEGIN RSA PRIVATE KEY-----MIIEowIBAAKCAQEAg0blRNV6cm3RTiivpzE8HR4JzKZRVIBZ7bxeNoMz0' \
'-----END RSA PRIVATE KEY-----'
vms = []
vm = {}
vm['role'] = 'master'
vm['public_ip'] = '192.168.1.10'
vm['user_name'] = 'user_name'
vm['public_ip'] = '127.0.0.1'
vm['semaphore'] = 'semaphore'
vms.append(vm)
vm['role'] = 'worker'
vm['public_ip'] = '192.168.1.11'
vm['public_ip'] = '127.0.0.1'
vm['role'] = 'worker'
vm['public_ip'] = '192.168.1.12'
vm['public_ip'] = '127.0.0.1'
vms.append(vm)
self.inventory_contents = yaml.dump( self.build_yml_inventory(vms),default_flow_style=False)
self.git_url = 'https://github.com/skoulouzis/playbooks.git'
self.git_url = 'https://github.com/QCDIS/playbooks.git'
self.playbook_name = 'get_ip_addresses.yaml'
self.semaphore_helper = SemaphoreHelper(self.semaphore_base_url, self.username, self.password)
......@@ -55,12 +56,22 @@ class TestTask(unittest.TestCase):
def testTask(self):
if SemaphoreHelper.service_is_up(self.semaphore_base_url):
project_id = self.semaphore_helper.create_project(self.project_name)
key_id = self.semaphore_helper.create_ssh_key(self.project_name, project_id, self.private_key)
inventory_id = self.semaphore_helper.create_inventory(self.project_name, project_id, key_id, self.inventory_contents)
repository_id = self.semaphore_helper.create_repository(self.project_name, project_id, key_id, self.git_url)
now = datetime.datetime.now()
key_name = 'key_name'+str(now)
key_id = self.semaphore_helper.create_ssh_key(key_name, project_id, self.private_key)
now = datetime.datetime.now()
inventory_name = 'inventory_name'+str(now)
inventory_id = self.semaphore_helper.create_inventory(inventory_name, project_id, key_id, self.inventory_contents)
now = datetime.datetime.now()
repository_name = 'repository_name'+str(now)
repository_id = self.semaphore_helper.create_repository(repository_name, project_id, key_id, self.git_url)
template_id = self.semaphore_helper.create_template(project_id, key_id, inventory_id, repository_id, self.playbook_name)
task_id = self.semaphore_helper.execute_task(project_id, template_id, self.playbook_name)
env_vars = {"SOME_VAR": "the contents"}
now = datetime.datetime.now()
environment_name = 'environment_name_name_'+str(now)
environment_id = self.semaphore_helper.create_environment(project_id,environment_name,env_vars)
task_id = self.semaphore_helper.execute_task(project_id, template_id, self.playbook_name,environment_id=environment_id)
for x in range(0, 2):
task = self.semaphore_helper.get_task(project_id, task_id)
print(task)
......@@ -68,20 +79,20 @@ class TestTask(unittest.TestCase):
# print(task_output)
sleep(1)
task_id = self.semaphore_helper.execute_task(project_id, template_id, self.playbook_name)
for x in range(0, 2):
task = self.semaphore_helper.get_task(project_id, task_id)
print(task)
# task_output = self.semaphore_helper.get_task_output(project_id, task_id)
# print(task_output)
sleep(1)
task_id = self.semaphore_helper.execute_task(project_id, template_id, 'mount.yaml')
for x in range(0, 2):
task = self.semaphore_helper.get_task(project_id, task_id)
print(task)
# task_output = self.semaphore_helper.get_task_output(project_id, task_id)
# print(task_output)
sleep(1)
# task_id = self.semaphore_helper.execute_task(project_id, template_id, self.playbook_name)
# for x in range(0, 2):
# task = self.semaphore_helper.get_task(project_id, task_id)
# print(task)
# # task_output = self.semaphore_helper.get_task_output(project_id, task_id)
# # print(task_output)
# sleep(1)
# task_id = self.semaphore_helper.execute_task(project_id, template_id, 'get_ip_addresses.yaml')
# for x in range(0, 2):
# task = self.semaphore_helper.get_task(project_id, task_id)
# print(task)
# # task_output = self.semaphore_helper.get_task_output(project_id, task_id)
# # print(task_output)
# sleep(1)
......@@ -89,7 +100,7 @@ class TestTask(unittest.TestCase):
inventory = {}
all = {}
vars = {'ansible_ssh_common_args':'-o StrictHostKeyChecking=no'}
vars['ansible_ssh_user'] = vms[0]['user_name']
vars['ansible_ssh_user'] = vms[0]['semaphore']
children = {}
for vm in vms:
role = vm['role']
......
sure_tosca-client_python_stubs/test/test_default_api.py
View file @ 21ce2a62
......@@ -14,9 +14,10 @@ from __future__ import absolute_import
import os
import unittest
from urllib.error import HTTPError
import urllib3
from urllib.request import urlopen
import ssl
import sure_tosca_client
from sure_tosca_client import Configuration, ApiClient
from sure_tosca_client.api.default_api import DefaultApi # noqa: E501
......@@ -28,7 +29,7 @@ class TestDefaultApi(unittest.TestCase):
def setUp(self):
configuration = Configuration()
configuration.host = 'http://127.0.0.1:8081/tosca-sure/1.0.0' #Make sure we don't have '/' on the end of url
if self.service_is_up(configuration.host):
if self.check_service_is_up(configuration.host):
configuration.verify_ssl = False
api_client = ApiClient(configuration=configuration)
self.api = sure_tosca_client.api.default_api.DefaultApi(api_client=api_client) # noqa: E501
......@@ -190,12 +191,17 @@ class TestDefaultApi(unittest.TestCase):
return file_id
def service_is_up(self, url):
def check_service_is_up(self, url):
code = None
try:
http = urllib3.PoolManager()
r = http.request('HEAD', url)
response = urlopen(url, context=ssl._create_unverified_context())
print(response)
# http = urllib3.PoolManager()
# r = http.request('HEAD', url,context=ctx)
except Exception as e:
if isinstance(e, HTTPError) and e.code == 401:
return True
return False
return True
......
sure_tosca-flask-server/.dockerignore
View file @ 21ce2a62
......@@ -3,7 +3,7 @@
README.md
tox.ini
git_push.sh
test-requirements.txt
#test-requirements.txt
setup.py
# Byte-compiled / optimized / DLL files
......
sure_tosca-flask-server/Dockerfile
View file @ 21ce2a62
FROM python:3.7-buster
FROM python:3.6-buster
RUN mkdir -p /usr/src/app
WORKDIR /usr/src/app
COPY requirements.txt /usr/src/app/
COPY test-requirements.txt /usr/src/app/
RUN pip install --no-cache-dir -r requirements.txt
#RUN pip install --no-cache-dir -r test-requirements.txt
COPY . /usr/src/app
#RUN python -m unittest discover
#RUN pip uninstall -y --no-cache-dir -r test-requirements.txt
EXPOSE 8081
ENTRYPOINT ["python"]
......
sure_tosca-flask-server/requirements.txt
View file @ 21ce2a62
......@@ -6,3 +6,4 @@ swagger-ui-bundle==0.0.6
tinydb==3.15.2
tosca-parser==2.0.0
flask-restx==0.2.0
werkzeug==1.0.1
\ No newline at end of file
sure_tosca-flask-server/sure_tosca/service/tosca_template_service.py
View file @ 21ce2a62
import json
import copy
import logging
import os
import tempfile
import time
import uuid
from builtins import print
from functools import reduce
import copy
import yaml
from tinydb import TinyDB, Query
from tinydb.middlewares import CachingMiddleware
from tinydb.storages import MemoryStorage
from toscaparser.functions import GetAttribute
from toscaparser.tosca_template import ToscaTemplate
from werkzeug.datastructures import FileStorage
......@@ -20,7 +16,6 @@ from sure_tosca.models.base_model_ import Model
from sure_tosca.models.node_template import NodeTemplateModel as NodeTemplateModel
from sure_tosca.models.node_template_map import NodeTemplateMapModel
from sure_tosca.models.tosca_template import ToscaTemplateModel as ToscaTemplateModel
from sure_tosca.service import tosca_helper
# db = TinyDB(storage=CachingMiddleware(MemoryStorage))
......@@ -110,7 +105,7 @@ def save(file: FileStorage):
purge_all_tables()
dictionary = yaml.safe_load(file.stream)
# dictionary = yaml.load(file.stream)
logger.info("tosca template: \n" + str(yaml.dump(dictionary)))
logger.debug("tosca template: \n" + str(yaml.dump(dictionary)))
# print(yaml.dump(dictionary))
tosca_template = get_tosca_template(dictionary)
# all_custom_def = tosca_template.nodetemplates[0].custom_def
......
sure_tosca-flask-server/sure_tosca/service/tosca_type_service.py
View file @ 21ce2a62
from sure_tosca.models.node_template import NodeTemplateModel # noqa: E501
from sure_tosca.models.node_template_map import NodeTemplateMapModel # noqa: E501
from sure_tosca.models.tosca_template import ToscaTemplateModel # noqa: E501
def add_type_definition(definition_map):
return None
......
sure_tosca-flask-server/sure_tosca/test/test_default_controller.py
View file @ 21ce2a62
......@@ -2,25 +2,59 @@
from __future__ import absolute_import
import os
import tempfile
import requests
from flask import json
from six import BytesIO
import os
from sure_tosca.models.node_template import NodeTemplateModel # noqa: E501
from sure_tosca.models.topology_template import TopologyTemplateModel # noqa: E501
from sure_tosca.models.tosca_template import ToscaTemplateModel # noqa: E501
from sure_tosca.test import BaseTestCase
class TestDefaultController(BaseTestCase):
"""DefaultController integration test stubs"""
def test_upload(self):
"""Test case for get_all_ancestor_properties
"""
id_example = self.upload_file('https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/TIC.yaml')
doc_id = int(id_example)
self.assertIsInstance(doc_id,int)
# id_example = self.upload_file('https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/application_example_planed.yaml')
# doc_id = int(id_example)
self.assertIsInstance(doc_id,int)
id_example = self.upload_file(
'https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/application_example_provisioned.yaml')
doc_id = int(id_example)
self.assertIsInstance(doc_id,int)
id_example = self.upload_file(
'https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/glusterFS.yaml')
doc_id = int(id_example)
self.assertIsInstance(doc_id,int)
id_example = self.upload_file(
'https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/kubernetes.yaml')
doc_id = int(id_example)
self.assertIsInstance(doc_id,int)
id_example = self.upload_file(
'https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/lifeWatch_vre1.yaml')
doc_id = int(id_example)
self.assertIsInstance(doc_id,int)
id_example = self.upload_file(
'https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/topology.yaml')
doc_id = int(id_example)
self.assertIsInstance(doc_id,int)
def test_get_all_ancestor_properties(self):
"""Test case for get_all_ancestor_properties
"""
id_example = self.upload_file('application_example_2_topologies.yaml')
id_example = self.upload_file('https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/application_example_2_topologies.yaml')
self.assertTrue(id_example.strip().isdigit())
response = self.client.open(
'/tosca-sure/1.0.0/tosca_template/{id}/topology_template/node_templates/{node_name}/ancestors_properties'.format(
......@@ -35,7 +69,7 @@ class TestDefaultController(BaseTestCase):
"""
id_example = self.upload_file('application_example_2_topologies.yaml')
id_example = self.upload_file('https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/application_example_2_topologies.yaml')
self.assertTrue(id_example.strip().isdigit())
response = self.client.open(
'/tosca-sure/1.0.0/tosca_template/{id}/topology_template/node_templates/{node_name}/ancestors_types'.format(
......@@ -50,7 +84,7 @@ class TestDefaultController(BaseTestCase):
"""
id_example = self.upload_file('application_example_2_topologies.yaml')
id_example = self.upload_file('https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/application_example_2_topologies.yaml')
self.assertTrue(id_example.strip().isdigit())
response = self.client.open(
'/tosca-sure/1.0.0/tosca_template/{id}/topology_template/node_templates/{node_name}/ancestors_requirements'.format(
......@@ -66,7 +100,7 @@ class TestDefaultController(BaseTestCase):
"""
# query_string = [('anchors', 'anchors_example'), ('derived_from', 'derived_from_example')]
id_example = self.upload_file('application_example_2_topologies.yaml')
id_example = self.upload_file('https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/application_example_2_topologies.yaml')
self.assertTrue(id_example.strip().isdigit())
response = self.client.open(
'/tosca-sure/1.0.0/tosca_template/{id}/dsl_definitions'.format(id=id_example),
......@@ -78,7 +112,7 @@ class TestDefaultController(BaseTestCase):
"""
id_example = self.upload_file('application_example_2_topologies.yaml')
id_example = self.upload_file('https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/application_example_2_topologies.yaml')
self.assertTrue(id_example.strip().isdigit())
response = self.client.open(
'/tosca-sure/1.0.0/tosca_template/{id}/imports'.format(id=id_example),
......@@ -90,7 +124,7 @@ class TestDefaultController(BaseTestCase):
"""
id_example = self.upload_file('application_example_outputs.yaml')
id_example = self.upload_file('https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/application_example_outputs.yaml')
self.assertTrue(id_example.strip().isdigit())
response = self.client.open(
'/tosca-sure/1.0.0/tosca_template/{id}/topology_template/node_templates/{node_name}/outputs'.format(
......@@ -105,7 +139,7 @@ class TestDefaultController(BaseTestCase):
"""
id_example = self.upload_file('application_example_2_topologies.yaml')
id_example = self.upload_file('https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/application_example_2_topologies.yaml')
self.assertTrue(id_example.strip().isdigit())
response = self.client.open(
'/tosca-sure/1.0.0/tosca_template/{id}/topology_template/node_templates/{node_name}/properties'.format(
......@@ -120,7 +154,7 @@ class TestDefaultController(BaseTestCase):
"""
id_example = self.upload_file('application_example_2_topologies.yaml')
id_example = self.upload_file('https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/application_example_2_topologies.yaml')
self.assertTrue(id_example.strip().isdigit())
response = self.client.open(
'/tosca-sure/1.0.0/tosca_template/{id}/topology_template/node_templates/{node_name}/requirements'.format(
......@@ -135,7 +169,7 @@ class TestDefaultController(BaseTestCase):
"""
id_example = self.upload_file('application_example_2_topologies.yaml')
id_example = self.upload_file('https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/application_example_2_topologies.yaml')
self.assertTrue(id_example.strip().isdigit())
query_string = [('type_name', None),
('node_name', 'compute'),
......@@ -206,7 +240,7 @@ class TestDefaultController(BaseTestCase):
"""
id_example = self.upload_file('application_example_2_topologies.yaml')
id_example = self.upload_file('https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/application_example_2_topologies.yaml')
self.assertTrue(id_example.strip().isdigit())
response = self.client.open(
'/tosca-sure/1.0.0/tosca_template/{id}/topology_template/node_templates/{node_name}/type_name'.format(
......@@ -221,7 +255,7 @@ class TestDefaultController(BaseTestCase):
"""
id_example = self.upload_file('application_example_2_topologies.yaml')
id_example = self.upload_file('https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/application_example_2_topologies.yaml')
self.assertTrue(id_example.strip().isdigit())
response = self.client.open(
'/tosca-sure/1.0.0/tosca_template/{id}/topology_template/node_templates/{node_name}/derived_from'.format(
......@@ -236,7 +270,7 @@ class TestDefaultController(BaseTestCase):
"""
id_example = self.upload_file('application_example_2_topologies.yaml')
id_example = self.upload_file('https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/application_example_2_topologies.yaml')
self.assertTrue(id_example.strip().isdigit())
response = self.client.open(
'/tosca-sure/1.0.0/tosca_template/{id}/topology_template/node_templates/{node_name}/related'.format(
......@@ -251,7 +285,7 @@ class TestDefaultController(BaseTestCase):
"""
id_example = self.upload_file('application_example_2_topologies.yaml')
id_example = self.upload_file('https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/application_example_2_topologies.yaml')
self.assertTrue(id_example.strip().isdigit())
query_string = [('type_name', None),
('derived_from', None)]
......@@ -266,7 +300,7 @@ class TestDefaultController(BaseTestCase):
"""
id_example = self.upload_file('application_example_2_topologies.yaml')
id_example = self.upload_file('https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/application_example_2_topologies.yaml')
self.assertTrue(id_example.strip().isdigit())
response = self.client.open(
'/tosca-sure/1.0.0/tosca_template/{id}/topology_template'.format(id=id_example),
......@@ -280,7 +314,7 @@ class TestDefaultController(BaseTestCase):
"""
id_example = self.upload_file('application_example_2_topologies.yaml')
id_example = self.upload_file('https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/application_example_2_topologies.yaml')
self.assertTrue(id_example.strip().isdigit())
response = self.client.open(
'/tosca-sure/1.0.0/tosca_template/{id}'.format(id=id_example),
......@@ -294,7 +328,7 @@ class TestDefaultController(BaseTestCase):
"""
id_example = self.upload_file('application_example_2_topologies.yaml')
id_example = self.upload_file('https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/application_example_2_topologies.yaml')
self.assertTrue(id_example.strip().isdigit())
query_string = [('kind_of_type', 'interface_types'),
('has_interfaces', None),
......@@ -334,7 +368,7 @@ class TestDefaultController(BaseTestCase):
"""
id_example = self.upload_file('application_example_2_topologies.yaml')
id_example = self.upload_file('https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/application_example_2_topologies.yaml')
self.assertTrue(id_example.strip().isdigit())
properties = {'properties': {'cpu_frequency': '2 GHz'}}
response = self.client.open(
......@@ -354,7 +388,7 @@ class TestDefaultController(BaseTestCase):
"""
id_example = self.upload_file('application_example_2_topologies.yaml')
id_example = self.upload_file('https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/application_example_2_topologies.yaml')
self.assertTrue(id_example.strip().isdigit())
response = self.client.open(
'/tosca-sure/1.0.0/tosca_template/{id}/topology_template/node_templates'.format(id=id_example),
......@@ -370,7 +404,7 @@ class TestDefaultController(BaseTestCase):
"""
id_example = self.upload_file('application_example_2_topologies.yaml')
id_example = self.upload_file('https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/application_example_2_topologies.yaml')
self.assertTrue(id_example.strip().isdigit())
query_string = [('instance_name', 'instance_name_example'),
('operation_name', 'provision')]
......@@ -383,13 +417,8 @@ class TestDefaultController(BaseTestCase):
self.assertEqual(response.status_code, 200)
self.assertIsInstance(response.json, dict)
def upload_file(self, file_name):
tosca_path = "../../../TOSCA/"
input_tosca_file_path = tosca_path + '/' + file_name
if not os.path.exists(input_tosca_file_path):
tosca_path = "../TOSCA/"
input_tosca_file_path = tosca_path + '/' + file_name
def upload_file(self, url):
input_tosca_file_path = self.get_remote_tosca_file(url)
dir_path = os.path.dirname(os.path.realpath(__file__))
self.assertEqual(True, os.path.exists(input_tosca_file_path),
'Starting from: ' + dir_path + ' Input TOSCA file: ' + input_tosca_file_path + ' not found')
......@@ -405,6 +434,11 @@ class TestDefaultController(BaseTestCase):
content_type='multipart/form-data')
return response.data.decode('utf-8').replace('\n', '')
def get_remote_tosca_file(self, url):
tosca = requests.get(url)
input_tosca_file_path = os.path.join(tempfile.gettempdir(),'test_tosca_file.yaml')
open( input_tosca_file_path, 'wb').write(tosca.content)
return input_tosca_file_path
if __name__ == '__main__':
import unittest
......
sure_tosca-flask-server/sure_tosca/test/test_tosca_template_service.py
View file @ 21ce2a62
import os
import tempfile
from unittest import TestCase
import requests
from six import BytesIO
from werkzeug.datastructures import FileStorage
import logging
......@@ -15,16 +18,18 @@ formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(messag
h.setFormatter(formatter)
logger.addHandler(h)
logger.handler_set = True
base_url = 'https://raw.githubusercontent.com/QCDIS/sdia-tosca/master/examples/'
class Test(TestCase):
def test_upload(self):
file_names = ['application_example_outputs.yaml','application_example_2_topologies.yaml','lifeWatch_vre1.yaml',
'application_example_updated.yaml','compute.yaml','application_example_provisioned.yaml','topology.yaml',
'kubernetes.yaml','application_example_planed.yaml']
'kubernetes.yaml','TIC.yaml']
for file_name in file_names:
logger.info("Testing : " + str(file_name))
doc_id = tosca_template_service.save(self.upload_file(file_name))
doc_id = tosca_template_service.save(self.upload_file(base_url+file_name))
logger.info("doc_id : " + str(doc_id))
self.assertTrue (doc_id >= 0)
......@@ -39,7 +44,7 @@ class Test(TestCase):
def test_get_tosca_template_model_by_id(self):
doc_id = tosca_template_service.save(self.upload_file('application_example_updated.yaml'))
doc_id = tosca_template_service.save(self.upload_file(base_url+'application_example_updated.yaml'))
tosca_template_dict = tosca_template_service.get_tosca_template_dict_by_id(doc_id)
tosca_template_model = ToscaTemplateModel.from_dict(tosca_template_dict)
......@@ -47,17 +52,12 @@ class Test(TestCase):
self.assertIsNotNone(tosca_template_model.topology_template)
self.assertIsNotNone(tosca_template_model.topology_template.node_templates)
def upload_file(self, file_name):
tosca_path = "../../../TOSCA/"
input_tosca_file_path = tosca_path + '/' + file_name
if not os.path.exists(input_tosca_file_path):
tosca_path = "../TOSCA/"
input_tosca_file_path = tosca_path + '/' + file_name
def upload_file(self, url):
input_tosca_file_path = self.get_remote_tosca_file(url)
dir_path = os.path.dirname(os.path.realpath(__file__))
self.assertEqual(True, os.path.exists(input_tosca_file_path),
'Starting from: ' + dir_path + ' Input TOSCA file: ' + input_tosca_file_path + ' not found')
file = open(input_tosca_file_path, "r")
# with open(input_tosca_file_path, 'r') as file:
# contents = file.read()
......@@ -71,3 +71,10 @@ class Test(TestCase):
return FileStorage(stream=BytesIO(byte_contents), filename=input_tosca_file_path, name=input_tosca_file_path,
content_type=None,
content_length=None, headers=None)
def get_remote_tosca_file(self, url):
tosca = requests.get(url)
input_tosca_file_path = os.path.join(tempfile.gettempdir(),'test_tosca_file.yaml')
open( input_tosca_file_path, 'wb').write(tosca.content)
return input_tosca_file_path
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment