refactor planner

drip-planner/.idea/drip-planner.iml

+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$">
+      <excludeFolder url="file://$MODULE_DIR$/venv" />
+    </content>
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+  <component name="TestRunnerService">
+    <option name="PROJECT_TEST_RUNNER" value="Unittests" />
+  </component>
+</module>
\ No newline at end of file

drip-planner/__main__.py

+# To change this license header, choose License Headers in Project Properties.
+# To change this template file, choose Tools | Templates
+# and open the template in the editor.
+import json
+import os
+import os.path
+import tempfile
+import time
+import logging
+import pika
+import yaml
+import sys
+
+from planner.planner import Planner
+from service.spec_service import SpecService
+
+logger = logging.getLogger(__name__)
+
+
+# if not getattr(logger, 'handler_set', None):
+# logger.setLevel(logging.INFO)
+# h = logging.StreamHandler()
+# formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+# h.setFormatter(formatter)
+# logger.addHandler(h)
+# logger.handler_set = True
+
+
+def init_chanel(args):
+    global rabbitmq_host
+    if len(args) > 1:
+        rabbitmq_host = args[1]
+        queue_name = args[2]  # planner
+    else:
+        rabbitmq_host = '127.0.0.1'
+
+    connection = pika.BlockingConnection(pika.ConnectionParameters(host=rabbitmq_host))
+    channel = connection.channel()
+    channel.queue_declare(queue=queue_name)
+    return channel
+
+
+def start(this_channel):
+    this_channel.basic_qos(prefetch_count=1)
+    this_channel.basic_consume(queue=queue_name, on_message_callback=on_request)
+    logger.info(" [x] Awaiting RPC requests")
+    this_channel.start_consuming()
+
+
+def on_request(ch, method, props, body):
+    response = handle_delivery(body)
+
+    ch.basic_publish(exchange='',
+                     routing_key=props.reply_to,
+                     properties=pika.BasicProperties(correlation_id=
+                                                     props.correlation_id),
+                     body=str(response))
+    ch.basic_ack(delivery_tag=method.delivery_tag)
+
+
+def handle_delivery(message):
+    logger.info("Got: " + str(message))
+    try:
+        message = message.decode()
+    except (UnicodeDecodeError, AttributeError):
+        pass
+    parsed_json_message = json.loads(message)
+    owner = parsed_json_message['owner']
+    tosca_file_name = 'tosca_template'
+    tosca_template_json = parsed_json_message['toscaTemplate']
+
+    input_current_milli_time = lambda: int(round(time.time() * 1000))
+
+    # rabbit = DRIPLoggingHandler(host=rabbitmq_host, port=5672, user=owner)
+    # logger.addHandler(rabbit)
+
+    try:
+        tosca_folder_path = os.path.join(tempfile.gettempdir(), "planner_files", str(input_current_milli_time()))
+    except NameError:
+        import sys
+        millis = int(round(time.time() * 1000))
+        tosca_folder_path = os.path.dirname(os.path.abspath(sys.argv[0])) + os.path.join(tempfile.gettempdir(),
+                                                                                         "planner_files",
+                                                                                         str(millis))
+
+    if not os.path.exists(tosca_folder_path):
+        os.makedirs(tosca_folder_path)
+    input_tosca_file_path = os.path.join(tosca_folder_path, tosca_file_name + ".yml")
+    with open(input_tosca_file_path, 'w') as outfile:
+        outfile.write(yaml.dump(tosca_template_json))
+
+    conf = {'url': "http://host"}
+    spec_service = SpecService(conf)
+    test_planner = Planner(input_tosca_file_path, spec_service)
+    tosca_template = test_planner.resolve_requirements()
+    tosca_template = test_planner.set_infrastructure_specifications()
+    template_dict = tosca_util.get_tosca_template_2_topology_template_dictionary(tosca_template)
+    logger.info("template ----: \n" + yaml.dump(template_dict))
+
+    response = {'toscaTemplate': template_dict}
+    output_current_milli_time = int(round(time.time() * 1000))
+    response["creationDate"] = output_current_milli_time
+    response["parameters"] = []
+    if queue_name == "planner_queue":
+        logger.info("Planning")
+    logger.info("Returning plan")
+    logger.info("Output message:" + json.dumps(response))
+    return json.dumps(response)
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO)
+    if sys.argv[1] == "test_local":
+        tosca_path = "../TOSCA/"
+        input_tosca_file_path = tosca_path + '/application_example_2_topologies.yaml'
+        conf = {'url': "http://host"}
+        spec_service = SpecService(conf)
+        test_planner = Planner(input_tosca_file_path, spec_service)
+        test_tosca_template = test_planner.resolve_requirements()
+        test_tosca_template = test_planner.set_infrastructure_specifications()
+        template_dict = tosca_util.get_tosca_template_2_topology_template_dictionary(test_tosca_template)
+        logger.info("template ----: \n" + yaml.dump(template_dict))
+
+        try:
+            tosca_folder_path = os.path.join(tempfile.gettempdir(), tosca_path)
+        except NameError:
+            import sys
+
+            tosca_folder_path = os.path.dirname(os.path.abspath(sys.argv[0])) + os.path.join(tempfile.gettempdir(),
+                                                                                             tosca_path)
+        tosca_file_name = 'tosca_template'
+        input_tosca_file_path = tosca_path + '/application_example_2_topologies.yaml'
+
+        with open(input_tosca_file_path, 'w') as outfile:
+            outfile.write(yaml.dump(template_dict))
+
+        ToscaTemplate(input_tosca_file_path)
+
+        test_response = {'toscaTemplate': template_dict}
+        logger.info("Output message:" + json.dumps(test_response))
+    else:
+        logger.info("Input args: " + sys.argv[0] + ' ' + sys.argv[1] + ' ' + sys.argv[2])
+        channel = init_chanel(sys.argv)
+        global queue_name
+        queue_name = sys.argv[2]
+        start(channel)

drip-planner/planner/planner.py

+import logging
+from toscaparser.nodetemplate import NodeTemplate
+from toscaparser.tosca_template import ToscaTemplate
+from toscaparser.topology_template import TopologyTemplate
+
+import operator
+import matplotlib.pyplot as plt
+
+from service.simple_spec_alayzer import SimpleAnalyzer
+from util import tosca_helper
+
+
+class Planner:
+
+    def __init__(self, path, spec_service):
+        self.path = path
+        self.tosca_template = ToscaTemplate(path)
+        self.tosca_node_types = self.tosca_template.nodetemplates[0].type_definition.TOSCA_DEF
+        self.all_custom_def = self.tosca_template.nodetemplates[0].custom_def
+        self.all_node_types = {}
+        self.all_node_types.update(self.tosca_node_types.items())
+        self.all_node_types.update(self.all_custom_def.items())
+        self.required_nodes = []
+
+        self.spec_service = spec_service
+
+    def add_required_nodes_to_template(self, required_nodes):
+        for req_node in required_nodes:
+            node_template = tosca_helper.node_type_2_node_template(req_node, self.all_custom_def)
+            self.tosca_template.nodetemplates.append(node_template)
+        return self.tosca_template
+
+    def set_infrastructure_specifications(self):
+        # Start bottom up and (node without requirements leaf) and find the root of the graph.
+        # Get root performance, version requirements and set specs to required node
+        specification_analyzer = SimpleAnalyzer(self.tosca_template)
+        nodes_with_new_specifications = specification_analyzer.set_node_specifications()
+
+        for new_spec_node in nodes_with_new_specifications:
+            for index, node_in_temple in enumerate(self.tosca_template.nodetemplates):
+                if new_spec_node.name == node_in_temple.name:
+                    self.tosca_template.nodetemplates[index] = new_spec_node
+                    break
+
+        specification_analyzer = SimpleAnalyzer(self.tosca_template)
+        nodes_with_new_relationship_occurrences = specification_analyzer.set_relationship_occurrences()
+
+        added_node_names = []
+        for new_spec_occurrences in nodes_with_new_relationship_occurrences:
+            for index, node_in_temple in enumerate(self.tosca_template.nodetemplates):
+                if new_spec_occurrences.name == node_in_temple.name:
+                    added_node_names.append(new_spec_occurrences.name)
+                    self.tosca_template.nodetemplates[index] = new_spec_occurrences
+                    break
+
+        for new_spec_occurrences in nodes_with_new_relationship_occurrences:
+            if new_spec_occurrences.name not in added_node_names:
+                self.tosca_template.nodetemplates.append(new_spec_occurrences)
+        return self.tosca_template
+
+    def get_node_template_property(self, prop_key, node_prop_dict):
+        prop_value = self.spec_service.get_property(prop_key)
+        if prop_value:
+            return {prop_key: node_prop_dict}
+        else:
+            if 'required' in node_prop_dict and 'default' in node_prop_dict:
+                return node_prop_dict['default']
+            if 'constraints' in node_prop_dict:
+                constraints = node_prop_dict['constraints']
+                for constraint in constraints:
+
+                    if next(iter(constraint)) == 'greater_or_equal':
+                        return constraint[next(iter(constraint))]
+            return None
+
+    def resolve_requirements(self):
+        """ Resolve requirements. Go over all nodes and recursively resolve requirements till node has no
+        requirements  e.g. docker -> k8s -> cluster -> vm """
+        for node in self.tosca_template.nodetemplates:
+            self.add_required_nodes(node)
+        return self.add_required_nodes_to_template(self.required_nodes)
+
+    def add_required_nodes(self, node):
+        """Adds the required nodes in self.required_nodes for an input node."""
+        if isinstance(node, NodeTemplate):
+            logging.info('Resolving requirements for: ' + node.name)
+        elif isinstance(node, dict):
+            logging.info('Resolving requirements for: ' + str(next(iter(node))))
+        # Get all requirements for node.
+        all_requirements = self.get_all_requirements(node)
+        if not all_requirements:
+            logging.debug('Node: ' + tosca_helper.get_node_type_name(node) + ' has no requirements')
+            return
+        matching_node = self.find_best_node_for_requirements(all_requirements)
+
+        # Only add node that is not in node_templates
+        matching_node_type_name = next(iter(matching_node))
+        matching_node_template = tosca_helper.node_type_2_node_template(matching_node, self.all_custom_def)
+
+        # Add the requirements to the node we analyzed. e.g. docker needed host now we added the type and name of host
+        node = self.add_requirements(node, all_requirements, matching_node_template.name)
+        if not tosca_helper.contains_node_type(self.required_nodes, matching_node_type_name):
+            logging.info('  Adding: ' + str(matching_node_template.name))
+            self.required_nodes.append(matching_node)
+        # Find matching nodes for the new node's requirements
+        self.add_required_nodes(matching_node)
+
+    def get_all_requirements(self, node):
+        """Returns  all requirements for an input node including all parents requirements"""
+
+        node_type_name = tosca_helper.get_node_type_name(node)
+        logging.info('      Looking for requirements for node: ' + node_type_name)
+        # Get the requirements for this node from its definition e.g. docker: hostedOn k8s
+        def_type = self.all_node_types[node_type_name]
+        all_requirements = []
+        if 'requirements' in def_type.keys():
+            all_requirements = def_type['requirements']
+            logging.info('      Found requirements: ' + str(all_requirements) + ' for node: ' + node_type_name)
+
+        # Get the requirements for this node from the template. e.g. wordpress: connectsTo mysql
+        # node_requirements =  tosca_helper.get_node_requirements(node)
+        # if node_requirements:
+        #     all_requirements += node_requirements
+
+        # Put them all together
+        parent_requirements = tosca_helper.get_ancestors_requirements(node, self.all_node_types, self.all_custom_def)
+        parent_type = tosca_helper.get_node_type_name(node)
+        if parent_type and parent_requirements:
+            logging.info(
+                '       Adding to : ' + str(node_type_name) + '  parent requirements from: ' + str(parent_type))
+            if not all_requirements:
+                all_requirements += parent_requirements
+            else:
+                for all_requirement in all_requirements:
+                    for parent_requirement in parent_requirements:
+                        all_requirement_key =  next(iter(all_requirement))
+                        parent_requirement_key = next(iter(parent_requirement))
+                        if all_requirement_key != parent_requirement_key and all_requirement[all_requirement_key]['capability'] != parent_requirement[parent_requirement_key]['capability']:
+                            all_requirements.append(parent_requirement)
+
+            logging.debug('      all_requirements: ' + str(all_requirements))
+        return all_requirements
+
+    def get_node_types_by_capability(self, cap):
+        """Returns  all nodes that have the  capability: cap and have interfaces. This way we  distinguish between
+        'abstract' and 'concrete' """
+        candidate_nodes = {}
+        for tosca_node_type in self.all_node_types:
+            if tosca_node_type.startswith('tosca.nodes') and 'capabilities' in self.all_node_types[tosca_node_type]:
+                logging.debug('      Node: ' + str(tosca_node_type))
+                for caps in self.all_node_types[tosca_node_type]['capabilities']:
+                    logging.debug('          ' + str(
+                        self.all_node_types[tosca_node_type]['capabilities'][caps]['type']) + ' == ' + cap)
+                    if self.all_node_types[tosca_node_type]['capabilities'][caps]['type'] == cap:
+                        candidate_nodes[tosca_node_type] = self.all_node_types[tosca_node_type]
+                        logging.debug('          candidate_node: ' + str(tosca_node_type))
+
+        candidate_child_nodes = {}
+        for node in candidate_nodes:
+            candidate_child_nodes.update(self.get_child_nodes(node))
+
+        candidate_nodes.update(candidate_child_nodes)
+        capable_nodes = {}
+        # Only return the nodes that have interfaces. This means that they are not "abstract"
+        nodes_type_names_with_interface = tosca_helper.get_node_types_with_interface(candidate_nodes)
+        for type_name in nodes_type_names_with_interface:
+            capable_nodes[type_name] = candidate_nodes[type_name]
+        return capable_nodes
+
+    def find_best_node_for_requirements(self, all_requirements):
+        """Returns  the 'best' node for a set of requirements. Here we count the number of requiremets that the node
+        can cover and return the one which covers the most """
+        matching_nodes = {}
+        number_of_matching_requirement = {}
+        # Loop requirements to find nodes per requirement
+        for req in all_requirements:
+            if 'capability' in req[next(iter(req))]:
+                capability = req[next(iter(req))]['capability']
+                logging.info('  Looking for nodes with capability: ' + capability)
+                # Find all nodes in the definitions that have the capability: capability
+                capable_nodes = self.get_node_types_by_capability(capability)
+                if capable_nodes:
+                    # Add number of matching capabilities for each node.
+                    # Try to score matching_nodes to return one. The more requirements a node meets the better
+                    for node_type in capable_nodes:
+                        matching_requirement_count = 1
+                        if node_type not in number_of_matching_requirement:
+                            number_of_matching_requirement[node_type] = matching_requirement_count
+                        else:
+                            matching_requirement_count = number_of_matching_requirement[node_type]
+                            matching_requirement_count += 1
+
+                        number_of_matching_requirement[node_type] = matching_requirement_count
+                        logging.info('      Found: ' + str(node_type))
+                    matching_nodes.update(capable_nodes)
+                else:
+                    logging.error('Did not find any node with required capability: ' + str(capability))
+                    raise Exception('Did not find any node with required capability: ' + str(capability))
+        # if we only found 1 return it
+        if len(matching_nodes) == 1:
+            return matching_nodes
+
+        sorted_number_of_matching_requirement = sorted(number_of_matching_requirement.items(),
+                                                       key=operator.itemgetter(1))
+        index = len(sorted_number_of_matching_requirement) - 1
+        winner_type = next(iter(sorted_number_of_matching_requirement[index]))
+        return {winner_type: matching_nodes[winner_type]}
+
+    def get_child_nodes(self, parent_node_type_name):
+        child_nodes = {}
+        for tosca_node_type in self.all_node_types:
+            if tosca_node_type.startswith('tosca.nodes') and 'derived_from' in self.all_node_types[tosca_node_type]:
+                if parent_node_type_name == self.all_node_types[tosca_node_type]['derived_from']:
+                    child_nodes[tosca_node_type] = self.all_node_types[tosca_node_type]
+        return child_nodes
+
+    def add_requirements(self, node, missing_requirements, capable_node_name):
+        """Add the requirements to the node """
+        for req in missing_requirements:
+            req[next(iter(req))]['node'] = capable_node_name
+            if isinstance(node, NodeTemplate):
+                contains_requirement = False
+                for node_requirement in node.requirements:
+                    if node_requirement == req:
+                        contains_requirement = True
+                        break
+                if not contains_requirement:
+                    node.requirements.append(req)
+            elif isinstance(node, dict):
+                type_name = next(iter(node))
+                if 'requirements' not in node[type_name]:
+                    node[type_name]['requirements'] = []
+                node_requirements = node[type_name]['requirements']
+                contains_requirement = False
+                for node_requirement in node_requirements:
+                    if node_requirement == req:
+                        contains_requirement = True
+                        break
+                if not contains_requirement:
+                    node[type_name]['requirements'].append(req)
+        return node

drip-planner/service/simple_spec_alayzer.py

+import copy
+
+from toscaparser.nodetemplate import NodeTemplate
+from toscaparser.properties import Property
+
+import networkx as nx
+import logging
+
+from service.specification_analyzer import SpecificationAnalyzer
+from util import tosca_helper
+
+
+class SimpleAnalyzer(SpecificationAnalyzer):
+
+    def __init__(self, tosca_template):
+        super(SimpleAnalyzer, self).__init__(tosca_template)
+
+    def set_relationship_occurrences(self):
+        return_nodes = []
+        # nodes_with_occurrences_in_requirements = tosca_util.get_nodes_with_occurrences_in_requirements(
+        # self.tosca_template.nodetemplates)
+        orchestrator_nodes = tosca_helper.get_nodes_by_type('tosca.nodes.ARTICONF.Orchestrator',
+                                                          self.tosca_template.nodetemplates, self.all_node_types,
+                                                          self.all_custom_def)
+
+        if 'properties' in orchestrator_nodes[0].entity_tpl:
+            if 'masters_num' in orchestrator_nodes[0].entity_tpl['properties']:
+                masters_num = orchestrator_nodes[0].entity_tpl['properties']['masters_num']
+            if 'workers_num' in orchestrator_nodes[0].entity_tpl['properties']:
+                workers_num = orchestrator_nodes[0].entity_tpl['properties']['workers_num']
+        else:
+            masters_num = orchestrator_nodes[0].get_property_value('masters_num')
+            workers_num = orchestrator_nodes[0].get_property_value('workers_num')
+
+        topology_nodes = tosca_helper.get_nodes_by_type('tosca.nodes.ARTICONF.VM.topology',
+                                                      self.tosca_template.nodetemplates, self.all_node_types,
+                                                      self.all_custom_def)
+
+        # for requirement in topology_nodes[0].requirements:
+        #     requirement_dict = requirement[next(iter(requirement))]
+        #     if requirement_dict['capability'] == 'tosca.capabilities.ARTICONF.VM':
+        #         requirement_dict['occurrences'] = min_num_of_vm
+
+        vm_nodes = tosca_helper.get_nodes_by_type('tosca.nodes.ARTICONF.VM.Compute',
+                                                self.tosca_template.nodetemplates, self.all_node_types,
+                                                self.all_custom_def)
+        if vm_nodes:
+            for i in range(len(vm_nodes), masters_num):
+                old_vm_name = vm_nodes[0].name
+                new_vm = copy.deepcopy(vm_nodes[0])
+                new_vm_name = new_vm.name + '_' + str(i)
+                new_vm.name = new_vm_name
+                templates = new_vm.templates.pop(old_vm_name)
+                new_vm.templates[new_vm_name] = templates
+
+                return_nodes.append(new_vm)
+                for requirement in topology_nodes[0].requirements:
+                    requirement_key = next(iter(requirement))
+                    requirement_value = requirement[requirement_key]
+                    if requirement_value['capability'] == 'tosca.capabilities.ARTICONF.VM':
+                        new_requirement = copy.deepcopy(requirement)
+                        new_requirement[requirement_key]['node'] = new_vm.name
+                        topology_nodes[0].requirements.append(new_requirement)
+                        return_nodes.append(topology_nodes[0])
+                        break
+
+            for i in range(len(vm_nodes), workers_num + 1):
+                old_vm_name = vm_nodes[0].name
+                new_vm = copy.deepcopy(vm_nodes[0])
+                new_vm_name = new_vm.name + '_' + str(i)
+                new_vm.name = new_vm_name
+                templates = new_vm.templates.pop(old_vm_name)
+                new_vm.templates[new_vm_name] = templates
+                new_vm.templates[next(iter(new_vm.templates))]['properties']['role'] = "worker"
+
+                return_nodes.append(new_vm)
+                for requirement in topology_nodes[0].requirements:
+                    requirement_key = next(iter(requirement))
+                    requirement_value = requirement[requirement_key]
+                    if requirement_value['capability'] == 'tosca.capabilities.ARTICONF.VM':
+                        new_requirement = copy.deepcopy(requirement)
+                        new_requirement[requirement_key]['node'] = new_vm.name
+                        topology_nodes[0].requirements.append(new_requirement)
+                        return_nodes.append(topology_nodes[0])
+                        break
+
+        return return_nodes
+
+    def set_node_specifications(self):
+        nodes_to_implement_policies = self.get_nodes_to_implement_policy()
+        affected_nodes = []
+        for node_name in nodes_to_implement_policies:
+            policies = nodes_to_implement_policies[node_name]
+            affected_node = self.set_specs(node_name, policies, self.tosca_template.nodetemplates)
+            if affected_node:
+                affected_nodes.append(affected_node)
+
+        return affected_nodes
+
+    def get_nodes_to_implement_policy(self):
+        nodes_to_implement_policies = {}
+
+        for policy in self.tosca_template.policies:
+            for target in policy.targets:
+                for leaf in self.leaf_nodes:
+                    for affected_node_name in (nx.shortest_path(self.g, source=target, target=leaf)):
+                        if affected_node_name not in nodes_to_implement_policies:
+                            policy_list = []
+                            nodes_to_implement_policies[affected_node_name] = policy_list
+                        policy_list = nodes_to_implement_policies[affected_node_name]
+                        policy_list.append(policy.type)
+                        nodes_to_implement_policies[affected_node_name] = policy_list
+        return nodes_to_implement_policies
+
+    def set_node_properties_for_policy(self, affected_node, policies):
+        logging.info('Setting properties for: ' + str(affected_node.type))
+
+        ancestors_types = tosca_helper.get_all_ancestors_types(affected_node, self.all_node_types, self.all_custom_def)
+        # if 'tosca.nodes.ARTICONF.Orchestrator' in ancestors_types:
+        #     logging.info('Do Something')
+        properties = tosca_helper.get_all_ancestors_properties(affected_node, self.all_node_types,
+                                                             self.all_custom_def)
+
+        default_properties = {}
+        for node_property in properties:
+            default_property = self.get_defult_value(node_property)
+            if default_property:
+                default_properties[next(iter(default_property))] = default_property[next(iter(default_property))]
+
+        if default_properties:
+            for default_property in default_properties:
+                affected_node.get_properties_objects().append(default_property)
+
+            if 'properties' in affected_node.templates[next(iter(affected_node.templates))]:
+                for prop_name in affected_node.templates[next(iter(affected_node.templates))]['properties']:
+                    if 'required' not in affected_node.templates[next(iter(affected_node.templates))]['properties'][
+                        prop_name] and 'type' not in \
+                            affected_node.templates[next(iter(affected_node.templates))]['properties'][prop_name]:
+                        default_properties[prop_name] = \
+                            affected_node.templates[next(iter(affected_node.templates))]['properties'][prop_name]
+
+            affected_node.templates[next(iter(affected_node.templates))]['properties'] = default_properties
+
+            return affected_node
+        else:
+            return None
+
+    def set_specs(self, node_name, policies, nodes_in_template):
+        logging.info('node_name: ' + str(node_name) + ' will implement policies: ' + str(len(policies)))
+        for node in nodes_in_template:
+            if node.name == node_name:
+                affected_node = node
+                break
+
+        logging.info('node: ' + str(affected_node.type) + ' will implement policies: ' + str(len(policies)))
+        affected_node = self.set_node_properties_for_policy(affected_node, policies)
+
+        return affected_node
+
+    def get_defult_value(self, node_property):
+        if isinstance(node_property.value,
+                      dict) and 'required' in node_property.value and 'type' in node_property.value:
+            if node_property.value['required']:
+                default_prop = {}
+                if 'default' in node_property.value:
+                    if node_property.value['type'] == 'integer':
+                        default_prop = int(node_property.value['default'])
+                    else:
+                        default_prop = str(node_property.value['default'])
+                elif 'constraints' in node_property.value:
+                    constraints = node_property.value['constraints']
+                    for constraint in constraints:
+                        for constraint_key in constraint:
+                            if 'equal' in constraint_key:
+                                if node_property.value['type'] == 'integer':
+                                    default_prop = int(constraint[constraint_key])
+                                else:
+                                    default_prop = str(constraint[constraint_key])
+                name = node_property.name
+                node_property = {name: default_prop}
+                return node_property
+        return None

drip-planner/service/spec_service.py

+class SpecService:
+
+    def __init__(self, conf):
+        self.configuration = conf
+
+    def get_property(self, prop_key):
+        return None

drip-planner/service/specification_analyzer.py

+from abc import abstractmethod, ABCMeta
+
+from toscaparser.tosca_template import ToscaTemplate
+import networkx as nx
+
+import matplotlib.pyplot as plt
+
+
+class SpecificationAnalyzer(metaclass=ABCMeta):
+
+    def __init__(self, tosca_template):
+        self.tosca_template = tosca_template
+        self.tosca_node_types = self.tosca_template.nodetemplates[0].type_definition.TOSCA_DEF
+        self.all_custom_def = self.tosca_template.nodetemplates[0].custom_def
+        self.all_node_types = {}
+        self.all_node_types.update(self.tosca_node_types.items())
+        self.all_node_types.update(self.all_custom_def.items())
+        self.required_nodes = []
+
+        self.g = self.build_graph(self.tosca_template.nodetemplates)
+
+        self.root_nodes = []
+        self.leaf_nodes = []
+        for node_name, degree in self.g.in_degree():
+            if degree == 0:
+                self.root_nodes.append(node_name)
+        for node_name, degree in self.g.out_degree():
+            if degree == 0:
+                self.leaf_nodes.append(node_name)
+
+    def build_graph(self, node_templates):
+        graph = nx.DiGraph()
+        for node in node_templates:
+            graph.add_node(node.name, attr_dict=node.entity_tpl)
+            for req in node.requirements:
+                req_name = next(iter(req))
+                req_node_name = req[req_name]['node']
+                if 'relationship' in req[req_name] and 'type' in req[req_name]['relationship']:
+                    relationship_type = req[req_name]['relationship']['type']
+                else:
+                    if 'relationship' not in req[req_name]:
+                        relationship_type = 'tosca.relationships.DependsOn'
+                    else:
+                        relationship_type = req[req_name]['relationship']
+                graph.add_edge(node.name, req_node_name, relationship=relationship_type)
+
+        nx.draw(graph, with_labels=True)
+        plt.savefig("/tmp/graph.png")
+        # plt.show()
+        return graph
+
+    @abstractmethod
+    def set_node_specifications(self):
+        raise NotImplementedError('Must implement upload in subclasses')
+
+    @abstractmethod
+    def set_relationship_occurrences(self):
+        raise NotImplementedError('Must implement upload in subclasses')

drip-planner/setup.py

+from setuptools import setup, find_packages
+
+setup(
+    name='drip_planner2',
+    version='0.1',
+    packages=find_packages(),
+
+    # Declare your packages' dependencies here, for eg:
+    install_requires=['matplotlib==3.1.1', 'pika==1.1.0', 'tosca-parser==1.6.0', 'names==0.3.0', 'networkx==2.4',
+                      'matplotlib==3.1.1'],
+
+    # Fill in these to make your Egg ready for upload to
+    # PyPI
+    author='S. Koulouzis',
+    author_email='',
+
+    # summary = 'Just another Python package for the cheese shop',
+    url='',
+    license='',
+    long_description='Long description of the package',
+
+    # could also include long_description, download_url, classifiers, etc.
+
+)

drip-planner/test/test_planner.py

+import json
+import logging
+import os
+import os.path
+import tempfile
+import time
+import unittest
+
+import yaml
+from toscaparser.tosca_template import ToscaTemplate
+
+from planner.planner import Planner
+from service.spec_service import SpecService
+
+
+class MyTestCase(unittest.TestCase):
+
+    def test_something(self):
+        logger = logging.getLogger(__name__)
+        tosca_path = "../../TOSCA/"
+        input_tosca_file_path = tosca_path + '/application_example_2_topologies.yaml'
+
+        dir_path = os.path.dirname(os.path.realpath(__file__))
+        print(dir_path)
+
+        self.assertEqual(True, os.path.exists(input_tosca_file_path),
+                         "Input TOSCA file: " + input_tosca_file_path + " not found")
+
+        conf = {'url': "http://host"}
+        spec_service = SpecService(conf)
+        test_planner = Planner(input_tosca_file_path, spec_service)
+        test_tosca_template = test_planner.resolve_requirements()
+        test_tosca_template = test_planner.set_infrastructure_specifications()
+        template_dict = tosca_util.get_tosca_template_2_topology_template_dictionary(test_tosca_template)
+        logger.info("template ----: \n" + yaml.dump(template_dict))
+
+        try:
+            tosca_folder_path = os.path.join(tempfile.gettempdir(), tosca_path)
+        except NameError:
+            import sys
+
+            tosca_folder_path = os.path.dirname(os.path.abspath(sys.argv[0])) + os.path.join(tempfile.gettempdir(),
+                                                                                             tosca_path)
+        tosca_file_name = 'tosca_template'
+        input_tosca_file_path = tosca_path + '/application_example_output.yaml'
+
+        with open(input_tosca_file_path, 'w') as outfile:
+            outfile.write(yaml.dump(template_dict))
+
+        ToscaTemplate(input_tosca_file_path)
+
+        test_response = {'toscaTemplate': template_dict}
+
+        response = {'toscaTemplate': template_dict}
+        output_current_milli_time = int(round(time.time() * 1000))
+        response["creationDate"] = output_current_milli_time
+        response["parameters"] = []
+        print("Output message:" + json.dumps(response))
+        self.assertEqual(True, True)
+
+
+if __name__ == '__main__':
+    unittest.main()

drip-planner/util/tosca_helper.py

+import copy
+from itertools import chain
+
+from toscaparser import tosca_template
+from toscaparser.elements.nodetype import NodeType
+from toscaparser.nodetemplate import NodeTemplate
+
+import yaml
+import logging
+
+# TOSCA template key names
+SECTIONS = (DEFINITION_VERSION, DEFAULT_NAMESPACE, TEMPLATE_NAME,
+            TOPOLOGY_TEMPLATE, TEMPLATE_AUTHOR, TEMPLATE_VERSION,
+            DESCRIPTION, IMPORTS, DSL_DEFINITIONS, NODE_TYPES,
+            RELATIONSHIP_TYPES, RELATIONSHIP_TEMPLATES,
+            CAPABILITY_TYPES, ARTIFACT_TYPES, DATA_TYPES, INTERFACE_TYPES,
+            POLICY_TYPES, GROUP_TYPES, REPOSITORIES, INPUTS, NODE_TEMPLATES,
+            OUTPUTS, GROUPS, SUBSTITUION_MAPPINGS, POLICIES, TYPE, REQUIREMENTS,
+            ARTIFACTS, PROPERTIES, INTERFACES) = \
+    ('tosca_definitions_version', 'tosca_default_namespace',
+     'template_name', 'tosca_template', 'template_author',
+     'template_version', 'description', 'imports', 'dsl_definitions',
+     'node_types', 'relationship_types', 'relationship_templates',
+     'capability_types', 'artifact_types', 'data_types',
+     'interface_types', 'policy_types', 'group_types', 'repositories',
+     'inputs', 'node_templates', 'outputs', 'groups', 'substitution_mappings',
+     'policies', 'type', 'requirements', 'artifacts', 'properties', 'interfaces')
+
+node_type_key_names_to_remove = ['capabilities', 'derived_from']
+
+
+def get_node_type_name(node):
+    """Returns the node's type name as string"""
+    if isinstance(node, NodeTemplate):
+        if node.type:
+            if node.type and isinstance(node.type, str):
+                node_type = node.type
+            elif isinstance(node.type, NodeTemplate):
+                node_type = node.type.type
+        else:
+            node_type = None
+    elif isinstance(node, dict):
+        node_type = next(iter(node))
+    return node_type
+
+
+def get_node_requirements(node):
+    if isinstance(node, NodeTemplate):
+        node_requirements = node.requirements
+    elif isinstance(node, dict):
+        node_type_name = get_node_type_name(node)
+        if 'requirements' not in node[node_type_name]:
+            node[node_type_name]['requirements'] = {}
+        node_requirements = node[node_type_name]['requirements']
+    return node_requirements
+
+
+def get_parent_type(node):
+    if isinstance(node, NodeTemplate):
+        if node.parent_type:
+            parent_type = node.parent_type.type
+        else:
+            parent_type = None
+    elif isinstance(node, dict):
+        parent_type = node[next(iter(node))]['derived_from']
+    return parent_type
+
+
+def get_node_type_requirements(type_name, all_nodes):
+    """Returns  the requirements for an input node as described in the template not in the node's definition """
+    def_type = all_nodes[type_name]
+    if 'requirements' in def_type.keys():
+        return def_type['requirements']
+    return None
+
+
+def get_ancestors_requirements(node, all_nodes, all_custom_def, parent_requirements=None):
+    """Recursively get all requirements all the way to the ROOT including the input node's"""
+    if not parent_requirements:
+        parent_requirements = []
+    if isinstance(node, NodeTemplate):
+        # If node has parent and parent has requirements add them
+        if node.parent_type and node.parent_type.requirements:
+            if isinstance(node.parent_type.requirements, dict):
+                parent_requirements.append(node.parent_type.requirements)
+            elif isinstance(node.parent_type.requirements, list):
+                parent_requirements.extend(node.parent_type.requirements)
+            # Make parent type to NodeTemplate to continue
+            if node.parent_type.type:
+                parent_template = node_type_2_node_template({'name': all_nodes[node.parent_type.type]}, all_custom_def)
+            if parent_template:
+                get_ancestors_requirements(parent_template, all_nodes, parent_requirements)
+    elif isinstance(node, dict):
+        node_type_name = get_node_type_name(node)
+        node_template = node_type_2_node_template({'name': all_nodes[node_type_name]}, all_custom_def)
+        return get_ancestors_requirements(node_template, all_nodes, all_custom_def, parent_requirements)
+    return parent_requirements
+
+
+def get_node_types_with_interface(nodes):
+    node_types_with_interface = []
+    for node_name in nodes:
+        if 'interfaces' in nodes[node_name].keys() and 'tosca.nodes.Root' != node_name:
+            node_types_with_interface.append(node_name)
+    return node_types_with_interface
+
+
+def node_type_2_node_template(node_type, all_custom_def):
+    node_template_dict = {}
+    type_name = next(iter(node_type))
+    node_type_array = type_name.split(".")
+    name = node_type_array[len(node_type_array) - 1].lower()
+    node_template_dict[name] = node_type[next(iter(node_type))].copy()
+    node_template_dict[name]['type'] = type_name
+
+    for name_to_remove in node_type_key_names_to_remove:
+        if name_to_remove in node_template_dict[name]:
+            node_template_dict[name].pop(name_to_remove)
+
+    if 'type' in node_type[next(iter(node_type))]:
+        node_type[next(iter(node_type))].pop('type')
+
+    node_template = NodeTemplate(name, node_template_dict, node_type)
+    # For some reason the tosca.nodes.ARTICONF.Orchestrator doesn't  have all definitions so we need to add them
+    # manually. We get 'toscaparser.common.exception.InvalidTypeError: Type "tosca.nodes.ARTICONF.Orchestrator"
+    # is not a valid type.'
+    if len(node_template.custom_def) < len(all_custom_def):
+        for def_key in all_custom_def:
+            if isinstance(def_key, dict):
+                node_template.custom_def.update(def_key)
+            else:
+                node_template.custom_def[def_key] = all_custom_def[def_key]
+
+    return node_template
+
+
+def get_tosca_template_2_topology_template_dictionary(template):
+    yaml_str = tosca_template2_yaml(template)
+    tosca_template_dict = yaml.load(yaml_str, Loader=yaml.FullLoader)
+    this_tosca_template = tosca_template_dict['tosca_template']
+    tosca_template_dict.pop('tosca_template')
+    tosca_template_dict['topology_template'] = this_tosca_template
+
+    if template.policies and 'policies' not in tosca_template_dict['topology_template']:
+        policies_list = []
+        for policy in template.policies:
+            policy_dict = {policy.name: policy.entity_tpl}
+            policies_list.append(policy_dict)
+        tosca_template_dict['topology_template']['policies'] = policies_list
+    return tosca_template_dict
+
+
+def contains_node_type(node_types_list, node_type_name):
+    if node_types_list is None:
+        return False
+    for node_type in node_types_list:
+        if isinstance(node_type, NodeTemplate):
+            type_name = node_type.type
+        elif isinstance(node_type, dict):
+            type_name = next(iter(node_type))
+        if type_name == node_type_name:
+            return True
+    return False
+
+
+def get_node_properties(node):
+    node_type_name = get_node_type_name(node)
+    return node[node_type_name]['properties']
+
+
+def set_node_properties(node, properties):
+    node_type_name = get_node_type_name(node)
+    node[node_type_name]['properties'] = properties
+    return node
+
+
+def get_nodes_by_type(node_type, nodes, all_node_types, all_custom_def):
+    nodes_by_type = []
+    for node in nodes:
+        if node.type == node_type:
+            nodes_by_type.append(node)
+            break
+        elif node_type in get_all_ancestors_types(node, all_node_types, all_custom_def):
+            nodes_by_type.append(node)
+    return nodes_by_type
+
+
+def get_all_ancestors_types(child_node, all_node_types, all_custom_def, ancestors_types=None):
+    if not ancestors_types:
+        ancestors_types = [get_node_type_name(child_node)]
+    parent_type = get_parent_type(child_node)
+    if parent_type:
+        ancestors_types.append(parent_type)
+        parent_type = node_type_2_node_template({'name': all_node_types[parent_type]}, all_custom_def)
+        get_all_ancestors_types(parent_type, all_node_types, all_custom_def, ancestors_types)
+    return ancestors_types
+
+
+def get_all_ancestors_properties(node, all_nodes, all_custom_def, ancestors_properties=None, ancestors_types=None):
+    if not ancestors_properties:
+        ancestors_properties = []
+        ancestors_properties_names = []
+        node_prop_names = []
+        if node.get_properties_objects():
+            for node_prop in node.get_properties_objects():
+                node_prop_names.append(node_prop.name)
+                ancestors_properties.append(node_prop)
+    if not ancestors_types:
+        ancestors_types = get_all_ancestors_types(node, all_nodes, all_custom_def)
+
+    for ancestors_type in ancestors_types:
+        ancestor = node_type_2_node_template({'name': all_nodes[ancestors_type]}, all_custom_def)
+        if ancestor.get_properties_objects():
+            for ancestor_prop in ancestor.get_properties_objects():
+                if ancestor_prop.name not in ancestors_properties_names and ancestor_prop.name not in node_prop_names:
+                    ancestors_properties_names.append(ancestor_prop.name)
+                    ancestors_properties.append(ancestor_prop)
+
+    return ancestors_properties
+
+
+def get_nodes_with_occurrences_in_requirements(topology_nodes):
+    nodes_with_occurrences_in_requirement = []
+    for node in topology_nodes:
+        for requirement in node.requirements:
+            requirement_dict = requirement[next(iter(requirement))]
+            if 'occurrences' in requirement_dict:
+                nodes_with_occurrences_in_requirement.append(node)
+                break
+    return nodes_with_occurrences_in_requirement
+
+
+def tosca_template2_yaml(tosca_template):
+    topology_dict = {DEFINITION_VERSION: tosca_template.version, IMPORTS: tosca_template._tpl_imports(),
+                     DESCRIPTION: tosca_template.description, TOPOLOGY_TEMPLATE: {}}
+    topology_dict[TOPOLOGY_TEMPLATE][NODE_TEMPLATES] = {}
+    node_templates = tosca_template.nodetemplates
+    for node_template in node_templates:
+        node_template_dict = get_node_template_dict(node_template)
+        topology_dict[TOPOLOGY_TEMPLATE][NODE_TEMPLATES][node_template.name] = node_template_dict
+
+    # If we don't add this then dump uses references for the same dictionary entries i.e. '&id001'
+    yaml.Dumper.ignore_aliases = lambda *args: True
+    return yaml.dump(topology_dict, default_flow_style=False)
+
+
+def get_node_template_dict(node_template):
+    node_template_dict = {TYPE: node_template.type}
+    #        node_template_dict[REQUIREMENTS] = {}
+    if node_template.requirements:
+        node_template_dict[REQUIREMENTS] = node_template.requirements
+    #        if node_template.interfaces:
+    #            interfaces = {}
+    #            for interface in node_template.interfaces:
+    #                interfaces[interface.type] = {}
+    #                interfaces[interface.type][interface.name] = interface.implementation
+
+    #        print( node_template.templates[node_template.name] )
+    if ARTIFACTS in node_template.templates[node_template.name].keys():
+        node_template_dict[ARTIFACTS] = node_template.templates[node_template.name][ARTIFACTS]
+    if PROPERTIES in node_template.templates[node_template.name].keys():
+        node_template_dict[PROPERTIES] = node_template.templates[node_template.name][PROPERTIES]
+    if INTERFACES in node_template.templates[node_template.name].keys():
+        node_template_dict[INTERFACES] = node_template.templates[node_template.name][INTERFACES]
+    #        print(dir(node_template))
+    #        print(node_template.templates)
+
+    return node_template_dict