changed interface to include repo

TOSCA/types/interfaces.yml

@@ -71,6 +71,9 @@ interface_types:
       playbook:
         type: string
         required: false
+      repository:
+        type: string
+        required: false        
       
   tosca.interfaces.ARTICONF.Kubernetes:
     derived_from: tosca.interfaces.ARTICONF.Standard
@@ -78,6 +81,9 @@ interface_types:
       inventory_file:
         type: string
         required: false
+      repository:
+        type: string
+        required: false        
       playbook:
         type: string
         required: false

TOSCA/types/nodes.yaml

@@ -61,21 +61,26 @@ node_types:
             type: tosca.interfaces.ARTICONF.Kubernetes
             delete:
                 inputs:
-                    playbook: https://raw.githubusercontent.com/skoulouzis/CONF/develop/ansible_playbooks/delete_service.yml
+                    repository: https://github.com/skoulouzis/playbooks.git
+                    playbook: delete_service.yml
             create:
                 inputs:
-                    playbook: https://raw.githubusercontent.com/skoulouzis/CONF/develop/ansible_playbooks/create_service.yml   
+                    repository: https://github.com/skoulouzis/playbooks.git
+                    playbook: create_service.yml   
             scale:
                 inputs:
-                    playbook: https://raw.githubusercontent.com/skoulouzis/CONF/develop/ansible_playbooks/scale_service.yml  
+                    repository: https://github.com/skoulouzis/playbooks.git
+                    playbook: scale_service.yml  
                     replicas: 1
             autoscale:
                 inputs:
-                    playbook: https://raw.githubusercontent.com/skoulouzis/CONF/develop/ansible_playbooks/autoscale_service.yml  
-                    horizontal_pod_autoscaler: https://raw.githubusercontent.com/skoulouzis/CONF/develop/k8s/horizontal_pod_autoscaler.yml  
+                    repository: https://github.com/skoulouzis/playbooks.git                
+                    playbook: autoscale_service.yml  
+                    horizontal_pod_autoscaler: horizontal_pod_autoscaler.yml  
             info:
                 inputs:
-                    playbook: https://raw.githubusercontent.com/skoulouzis/CONF/develop/ansible_playbooks/get_info_service.yml
+                    repository: https://github.com/skoulouzis/playbooks.git   
+                    playbook: get_info_service.yml
                 
                     
                     
@@ -129,22 +134,18 @@ node_types:
             type: tosca.interfaces.ARTICONF.Standard
             create:
                 inputs:
-                    playbook: https://raw.githubusercontent.com/skoulouzis/CONF/develop/ansible_playbooks/k8s_install.yml  
+                    repository: https://github.com/skoulouzis/playbooks.git   
+                    playbook: k8s_install.yml  
             configure:
                 inputs:
-                    playbook: https://raw.githubusercontent.com/skoulouzis/CONF/develop/ansible_playbooks/k8s_dashboard.yaml 
+                    repository: https://github.com/skoulouzis/playbooks.git   
+                    playbook: k8s_dashboard.yaml 
             delete:
                 inputs:
-                    playbook: https://raw.githubusercontent.com/skoulouzis/CONF/develop/ansible_playbooks/k8s_delete.yaml                     
+                    repository: https://github.com/skoulouzis/playbooks.git   
+                    playbook: k8s_delete.yaml                     
                     
         
-  #osca.nodes.ARTICONF.docker.Orchestrator.Swarm:
-    #derived_from: tosca.nodes.ARTICONF.docker.Orchestrator
-    #description: swarm orchestrator
-    #interfaces:
-      #Standard:
-        #create: interfaces/playbooks/docker_swarm_install.yaml            
-        
   tosca.nodes.ARTICONF.VM.topology:
     derived_from: tosca.nodes.ARTICONF.Infrastructure
     description: topology of VMs

ansible_playbooks/inventory.yaml

+all:
+  vars:
+    ansible_ssh_user: ubunut
+    host_key_checking: False
+  children:
+    k8-master:
+      hosts:
+        192.168.1.10:
+    worker:
+      hosts:
+        192.168.1.11:
+        192.168.1.12:
+        192.168.1.13:

commons/src/main/java/nl/uva/sne/drip/commons/utils/Constants.java

@@ -28,6 +28,7 @@ public class Constants {
     public static final String DISK_SIZE = "disk_size";
     public static final String VM_OS = "os";
     public static final String VM_TOPOLOGY = "tosca.nodes.ARTICONF.VM.topology";
+    public static final String APPLICATION_TYPE = "tosca.nodes.ARTICONF.Application";
     public static final String CLOUD_STORM_INTERFACE = "tosca.interfaces.ARTICONF.CloudsStorm";
     public static final String ENCODED_FILE_DATATYPE = "tosca.datatypes.ARTICONF.encodedFile";
     public static final String CLOUD_STORM_FILES_ZIP_SUFIX = "cloudStromFiles.zip";

commons/src/main/java/nl/uva/sne/drip/commons/utils/ToscaHelper.java

@@ -131,6 +131,18 @@ public class ToscaHelper {
         }
     }
 
+    public List<NodeTemplateMap> getApplicationTemplates() throws ApiException {
+        try {
+            List<NodeTemplateMap> vmTopologyTemplates = api.getNodeTemplates(String.valueOf(id), APPLICATION_TYPE, null, null, null, null, null, null, null);
+            return vmTopologyTemplates;
+        } catch (ApiException ex) {
+            if (ex.getCode() == 404) {
+                return null;
+            }
+            throw ex;
+        }
+    }
+
     public List<NodeTemplateMap> getTemplateVMsForVMTopology(NodeTemplateMap nodeTemplateMap) throws ApiException {
         NodeTemplate nodeTemplate = nodeTemplateMap.getNodeTemplate();
         List<Map<String, Object>> requirements = nodeTemplate.getRequirements();

deployer/service/ansible_service.py

 import logging
+import os
+
+import yaml
 
-from service.polemarch_helper import PolemarchHelper
 from service.semaphore_helper import SemaphoreHelper
 
 logger = logging.getLogger(__name__)
@@ -15,24 +17,46 @@ if not getattr(logger, 'handler_set', None):
 
 class AnsibleService:
 
-    def __init__(self, polemarch_base_url=None,polemarch_username=None,polemarch_password=None,
-                 semaphore_base_url=None,semaphore_username=None,semaphore_password=None):
-        self.polemarch_base_url = polemarch_base_url
-        self.polemarch_username  = polemarch_username
-        self.polemarch_password = polemarch_password
+    def __init__(self, semaphore_base_url=None,semaphore_username=None,semaphore_password=None):
         self.semaphore_base_url = semaphore_base_url
         self.semaphore_username = semaphore_username
         self.semaphore_password = semaphore_password
 
 
 
-    def execute(self,nodes_pair):
+    def execute(self,nodes_pair,operation):
         vms = nodes_pair[0]
         orchestartor = nodes_pair[1]
-
-        # polemarch_helper = PolemarchHelper(self.polemarch_base_url,self.polemarch_username,self.polemarch_password)
-        # project = polemarch_helper.create_project('test1','GIT','https://github.com/skoulouzis/playbooks.git')
-        # inventory = polemarch_helper.add_inventory(project.id,'test_inventory',vms)
+        interfaces = orchestartor.node_template.interfaces
+        for interface in interfaces:
+            if interface == 'Standard':
+                standard = interfaces['Standard']
+
+        git_url = 'https://github.com/skoulouzis/playbooks.git'
+        private_key = '-----BEGIN RSA PRIVATE KEY-----6qFrczm3VYELw0Flw06Cf2Bza8rAVFnFqWpZJHLh7LFMt/U5ocn4df45NrE4UXo+hGoK7xWb/A' \
+              'zudkwDkSexIAUHx/yPsHXK0gIxGtpsAXzV+7Y+5bI4gsN+WAJgOASFi6YHJU1YuAkLkPk5Gqb5UGZn7DoS9cGFQKvLCxBQIDAQABAoIBA' \
+              'GXPM7ugfC-----END RSA PRIVATE KEY-----'
+        path = self.get_path('inventory.yaml')
+        with open(path, 'r') as stream:
+            data = yaml.load(stream)
+
+        inventory_contents = yaml.dump(data)
+        playbook_name = 'k8s_install.yml'
         semaphore_helper = SemaphoreHelper(self.semaphore_base_url,self.semaphore_username,self.semaphore_password)
-        semaphore_helper.create_project('test1')
-        pass
\ No newline at end of file
+
+        project_id = semaphore_helper.create_project(orchestartor.name)
+        key_id = semaphore_helper.create_ssh_key(orchestartor.name,project_id,private_key)
+        inventory_id = semaphore_helper.create_inventory(orchestartor.name, project_id, key_id, inventory_contents)
+        repository_id = semaphore_helper.create_repository(orchestartor.name,project_id,key_id,git_url)
+
+        template_id = semaphore_helper.create_template(project_id,key_id,inventory_id,repository_id,playbook_name)
+        task_id = semaphore_helper.execute_task(project_id, template_id, playbook_name)
+        pass
+
+    def get_path(self,file_name):
+        tosca_path = "../../../ansible_playbooks"
+        input_tosca_file_path = tosca_path + '/' + file_name
+        if not os.path.exists(input_tosca_file_path):
+            tosca_path = "../../ansible_playbooks"
+            input_tosca_file_path = tosca_path + '/' + file_name
+        return input_tosca_file_path
\ No newline at end of file

deployer/service/deploy_service.py

@@ -19,8 +19,7 @@ class DeployService:
 
         interface_types = tosca_helper.get_interface_types(source)
         if interface_types and 'Standard' in interface_types:
-            ansible_service = AnsibleService(self.polemarch_base_url,self.polemarch_username,self.polemarch_password,
-                                             self.semaphore_base_url,self.semaphore_username,self.semaphore_password)
+            ansible_service = AnsibleService(self.semaphore_base_url,self.semaphore_username,self.semaphore_password)
             ansible_service.execute(nodes_pair)
         # print(source)
         # print(target)

deployer/service/polemarch_helper.py

-import polemarch_client
-from polemarch_client import Configuration, ApiClient, api, ProjectCreateMaster, Empty, OneInventory, User, OneProject
-from polemarch_client.api import project_api
-
-
-class PolemarchHelper:
-
-    def __init__(self, polemarch_base_url,username,password):
-        self.init_polemarch_client(polemarch_base_url,username,password)
-
-
-    def init_polemarch_client(self,polemarch_base_url,username,password):
-        configuration = Configuration()
-        configuration.host = polemarch_base_url
-        configuration.username = username
-        configuration.password = password
-        api_client = ApiClient(configuration=configuration)
-
-        self.community_api = api.CommunityTemplateApi(api_client=api_client)
-        self.group_api = api.GroupApi(api_client=api_client)
-        self.history_api = api.HistoryApi(api_client=api_client)
-        self.hook_api = api.HookApi(api_client=api_client)
-        self.host_api = api.HostApi(api_client=api_client)
-        self.inventory_api = api.InventoryApi(api_client=api_client)
-        self.project_api = api.ProjectApi(api_client=api_client)
-        self.team_api = api.TeamApi(api_client=api_client)
-        self.user_api = api.UserApi(api_client=api_client)
-        users = self.user_api.user_list(username=username)
-        self.user = users.results[0]
-
-
-
-    def get_project(self,project_id):
-        project = self.project_api.project_get(project_id)
-        return project
-
-
-    def create_project(self,name,type,repository):
-        project_create_master = ProjectCreateMaster(name=name,type=type, repository=repository)
-        project_create_master =  self.project_api.project_add(project_create_master)
-        empty = Empty()
-        self.project_api.project_sync(project_create_master.id,empty)
-
-        # pr = OneProject(name=name,type=type, repository=repository)
-        # pr.owner = self.user
-        # view_data = {}
-        # playbook = {}
-        # playbook['help'] = 'some help'
-        # playbook['title'] = 'title'
-        # playbooks = {}
-        # playbooks['main.yml'] = playbook
-        # view_data['fields'] = playbooks
-        # pr.execute_view_data = view_data
-        # return project_create_master
-
-    def add_inventory(self,project_id,name,vms):
-        one_inventory = OneInventory()
-        one_inventory.name = name
-        one_inventory.owner = self.user
-        one_inventory =  self.project_api.project_inventory_add(project_id,one_inventory)
-        return one_inventory

deployer/service/semaphore_helper.py

-import polemarch_client
-from polemarch_client import Project
-from semaphore_client import Configuration, ApiClient, api, ProjectRequest, Login
-from polemarch_client.api import project_api
+from semaphore_client import Configuration, ApiClient, api, ProjectRequest, Login, Repository, Inventory, \
+    AccessKeyRequest, InventoryRequest, RepositoryRequest, TemplateRequest, Task
 
 
 class SemaphoreHelper:
@@ -16,10 +14,11 @@ class SemaphoreHelper:
         configuration.username = username
         configuration.password = password
         api_client = ApiClient(configuration=configuration)
-        self.authentication_api = api.AuthenticationApi(api_client=api_client)
-        login_body = Login(auth=username, password=password)
-        self.authentication_api.auth_login_post(login_body)
-        self.authentication_api.user_tokens_get()
+        if not api_client.cookie:
+            self.authentication_api = api.AuthenticationApi(api_client=api_client)
+            login_body = Login(auth=username, password=password)
+            self.authentication_api.auth_login_post(login_body)
+            tokens = self.authentication_api.user_tokens_get()
 
         self.default_api = api.DefaultApi(api_client=api_client)
         self.project_api = api.ProjectApi(api_client=api_client)
@@ -27,7 +26,73 @@ class SemaphoreHelper:
         self.projects_api = api.ProjectsApi(api_client=api_client)
 
 
-    def create_project(self,name):
+    def create_project_if_not_exists(self, name, git_url, private_key,inventory_contents,playbook_name):
+        the_project = self.create_project(name)
+        key = self.create_ssh_key(name,the_project.id,private_key)
+        inventory= self.create_inventory(name,the_project.id,key.id,inventory_contents)
+        repository = self.create_repository(name,the_project.id,key.id,git_url)
+
+        template = self.create_template(the_project.id,key.id,inventory.id,repository.id,playbook_name)
+
+        task = Task(template_id=template.id, playbook=playbook_name)
+        self.project_api.project_project_id_tasks_post(the_project.id, task)
+
+        tasks = self.project_api.project_project_id_tasks_get(the_project.id)
+        for task in tasks:
+            print(task)
+
+        return the_project
+
+    def find_projects_by_name(self, name):
+        projects = self.projects_api.projects_get()
+        projects_with_name = []
+        for project in projects:
+            if project.name == name:
+                projects_with_name.append(project)
+        return projects_with_name
+
+    def project_with_name_exists(self, name):
+        projects = self.find_projects_by_name(name)
+        if not projects:
+            return False
+        return True
+
+    def create_project(self, name):
         project_request = ProjectRequest(name=name)
-        res = self.projects_api.projects_post(project_request)
-        print(res)
+        self.projects_api.projects_post(project_request)
+        projects = self.find_projects_by_name(name)
+        return projects[len(projects)-1].id
+
+    def create_ssh_key(self, name, project_id, private_key):
+        key_request = AccessKeyRequest(name=name, type='ssh', project_id=project_id, secret=private_key)
+        self.project_api.project_project_id_keys_post(project_id, key_request)
+        keys = self.project_api.project_project_id_keys_get(project_id, name, 'asc', key_type='ssh')
+        return keys[len(keys) - 1].id
+
+    def create_inventory(self, name, project_id,ssh_key_id, inventory_contents):
+        inventory_request = InventoryRequest( name=name, project_id=project_id, inventory=inventory_contents,
+                                              ssh_key_id=ssh_key_id, type='static')
+        self.project_api.project_project_id_inventory_post(project_id,inventory_request)
+
+        inventories = self.project_api.project_project_id_inventory_get(project_id, name, 'asc')
+        return inventories[len(inventories) - 1].id
+
+    def create_repository(self, name, project_id, key_id, git_url):
+        repository_request = RepositoryRequest(name=name, project_id=project_id, git_url=git_url, ssh_key_id=key_id)
+        self.project_api.project_project_id_repositories_post(project_id, repository=repository_request)
+
+        repositories = self.project_api.project_project_id_repositories_get(project_id, name, 'asc')
+        return repositories[len(repositories) - 1].id
+
+    def create_template(self, project_id,key_id,inventory_id,repository_id,playbook_name):
+        template_request = TemplateRequest(ssh_key_id=key_id, project_id=project_id, inventory_id=inventory_id,
+                                           repository_id=repository_id, alias=playbook_name, playbook=playbook_name)
+        self.project_api.project_project_id_templates_post(project_id, template_request)
+        templates = self.project_api.project_project_id_templates_get(project_id, playbook_name, 'asc')
+        return templates[len(templates) - 1].id
+
+    def execute_task(self, project_id, template_id, playbook_name):
+        task = Task(template_id=template_id, playbook=playbook_name)
+        self.project_api.project_project_id_tasks_post(project_id, task)
+        tasks = self.project_api.project_project_id_tasks_get(project_id)
+        return tasks[len(tasks) - 1].id

manager/src/main/java/nl/uva/sne/drip/api/DeployerApiController.java

@@ -34,12 +34,14 @@ public class DeployerApiController implements DeployerApi {
     }
 
     @Override
-    public ResponseEntity<String> deployProvisionToscaTemplateByID(@ApiParam(value = "ID of topolog template to deploy", required = true) @PathVariable("id") String id) {
+    public ResponseEntity<String> deployProvisionToscaTemplateByID(
+            @ApiParam(value = "ID of topolog template to deploy", required = true) 
+            @PathVariable("id") String id) {
+        
         String accept = request.getHeader("Accept");
         if (accept != null && accept.contains("")) {
-
             try {
-                String planedYemplateId = dripService.deploy(id);
+                String planedYemplateId = dripService.deploy(id,null);
                 return new ResponseEntity<>(planedYemplateId, HttpStatus.OK);
 
             } catch (Exception ex) {

manager/src/main/java/nl/uva/sne/drip/service/DRIPService.java

@@ -116,18 +116,18 @@ public class DRIPService {
         if (vmTopologies == null || vmTopologies.isEmpty()) {
             throw new MissingVMTopologyException("ToscaTemplate: " + toscaTemplate + " has no VM Topologies");
         }
-        toscaTemplate = setDesieredSate(toscaTemplate, vmTopologies, NODE_STATES.RUNNING);
+        for (NodeTemplateMap vmTopology : vmTopologies) {
+            toscaTemplate = setDesieredSate(toscaTemplate, vmTopology, NODE_STATES.RUNNING);
+        }
         return execute(toscaTemplate, provisionerQueueName);
     }
 
     protected ToscaTemplate setDesieredSate(ToscaTemplate toscaTemplate,
-            List<NodeTemplateMap> nodes, NODE_STATES nodeState) throws IOException, JsonProcessingException, ApiException {
-        for (NodeTemplateMap node : nodes) {
-            NODE_STATES currentState = helper.getNodeCurrentState(node);
-            NODE_STATES desiredState = helper.getNodeDesiredState(node);
-            node = helper.setNodeDesiredState(node, nodeState);
-            toscaTemplate = helper.setNodeInToscaTemplate(toscaTemplate, node);
-        }
+            NodeTemplateMap node, NODE_STATES nodeState) throws IOException, JsonProcessingException, ApiException {
+        NODE_STATES currentState = helper.getNodeCurrentState(node);
+        NODE_STATES desiredState = helper.getNodeDesiredState(node);
+        node = helper.setNodeDesiredState(node, nodeState);
+        toscaTemplate = helper.setNodeInToscaTemplate(toscaTemplate, node);
         return toscaTemplate;
     }
 
@@ -140,8 +140,15 @@ public class DRIPService {
         }
     }
 
-    public String deploy(String id) throws JsonProcessingException, NotFoundException, IOException, ApiException, Exception {
+    public String deploy(String id, List<String> nodeNames) throws JsonProcessingException, NotFoundException, IOException, ApiException, Exception {
         ToscaTemplate toscaTemplate = initExecution(id);
+        //If no nodes are specified deploy all applications
+        if (nodeNames == null || nodeNames.isEmpty()) {
+            List<NodeTemplateMap> applicationTemplates = helper.getApplicationTemplates();
+            for (NodeTemplateMap applicationTemplate : applicationTemplates) {
+                toscaTemplate = setDesieredSate(toscaTemplate, applicationTemplate, NODE_STATES.RUNNING);
+            }
+        }
         return execute(toscaTemplate, deployerQueueName);
     }
 
@@ -159,7 +166,7 @@ public class DRIPService {
         if (nodeNames == null || nodeNames.isEmpty()) {
             List<NodeTemplateMap> vmTopologies = helper.getVMTopologyTemplates();
             for (NodeTemplateMap vmTopology : vmTopologies) {
-                toscaTemplate = setDesieredSate(toscaTemplate, vmTopologies, NODE_STATES.DELETED);
+                toscaTemplate = setDesieredSate(toscaTemplate, vmTopology, NODE_STATES.DELETED);
             }
             return execute(toscaTemplate, provisionerQueueName);
         } else {

manager/src/test/java/nl/uva/sne/drip/service/ServiceTests.java

@@ -434,8 +434,9 @@ public class ServiceTests {
                 assertNotNull(attributes);
                 Assert.assertTrue(attributes.containsKey("credential"));
                 assertNotNull(attributes.get("credential"));
+                toscaTemplate = dripService.setDesieredSate(toscaTemplate, vmTopology, Constants.NODE_STATES.RUNNING);
             }
-            toscaTemplate = dripService.setDesieredSate(toscaTemplate, vmTopologies, Constants.NODE_STATES.RUNNING);
+            
             Map<String, NodeTemplate> nodes = toscaTemplate.getTopologyTemplate().getNodeTemplates();
             Set<String> names = nodes.keySet();
             for (String name : names) {

semaphore-python-client-generated/semaphore_client/models/project.py

@@ -124,8 +124,8 @@ class Project(object):
         :param created: The created of this Project.  # noqa: E501
         :type: str
         """
-        if created is not None and not re.search(r'^\\d{4}-(?:0[0-9]{1}|1[0-2]{1})-[0-9]{2}T\\d{2}:\\d{2}:\\d{2}Z$', created):  # noqa: E501
-            raise ValueError(r"Invalid value for `created`, must be a follow pattern or equal to `/^\\d{4}-(?:0[0-9]{1}|1[0-2]{1})-[0-9]{2}T\\d{2}:\\d{2}:\\d{2}Z$/`")  # noqa: E501
+        # if created is not None and not re.search(r'^\\d{4}-(?:0[0-9]{1}|1[0-2]{1})-[0-9]{2}T\\d{2}:\\d{2}:\\d{2}Z$', created):  # noqa: E501
+        #     raise ValueError(r"Invalid value for `created`, must be a follow pattern or equal to `/^\\d{4}-(?:0[0-9]{1}|1[0-2]{1})-[0-9]{2}T\\d{2}:\\d{2}:\\d{2}Z$/`")  # noqa: E501
 
         self._created = created
 

semaphore-python-client-generated/semaphore_client/models/user.py

@@ -181,8 +181,8 @@ class User(object):
         :param created: The created of this User.  # noqa: E501
         :type: str
         """
-        if created is not None and not re.search(r'^\\d{4}-(?:0[0-9]{1}|1[0-2]{1})-[0-9]{2}T\\d{2}:\\d{2}:\\d{2}Z$', created):  # noqa: E501
-            raise ValueError(r"Invalid value for `created`, must be a follow pattern or equal to `/^\\d{4}-(?:0[0-9]{1}|1[0-2]{1})-[0-9]{2}T\\d{2}:\\d{2}:\\d{2}Z$/`")  # noqa: E501
+        # if created is not None and not re.search(r'^\\d{4}-(?:0[0-9]{1}|1[0-2]{1})-[0-9]{2}T\\d{2}:\\d{2}:\\d{2}Z$', created):  # noqa: E501
+        #     raise ValueError(r"Invalid value for `created`, must be a follow pattern or equal to `/^\\d{4}-(?:0[0-9]{1}|1[0-2]{1})-[0-9]{2}T\\d{2}:\\d{2}:\\d{2}Z$/`")  # noqa: E501
 
         self._created = created