added comments

drip_planner2/.idea/workspace.xml

@@ -2,12 +2,14 @@
 <project version="4">
   <component name="ChangeListManager">
     <list default="true" id="462ede19-adfe-472b-975e-fefefa973fe0" name="Default Changelist" comment="">
+      <change beforePath="$PROJECT_DIR$/../TOSCA/application_example_updated.yaml" beforeDir="false" afterPath="$PROJECT_DIR$/../TOSCA/application_example_updated.yaml" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/../TOSCA/types/nodes.yaml" beforeDir="false" afterPath="$PROJECT_DIR$/../TOSCA/types/nodes.yaml" afterDir="false" />
       <change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
       <change beforePath="$PROJECT_DIR$/src/planner/planner.py" beforeDir="false" afterPath="$PROJECT_DIR$/src/planner/planner.py" afterDir="false" />
       <change beforePath="$PROJECT_DIR$/src/planner/simple_spec_alayzer.py" beforeDir="false" afterPath="$PROJECT_DIR$/src/planner/simple_spec_alayzer.py" afterDir="false" />
       <change beforePath="$PROJECT_DIR$/src/planner/specification_analyzer.py" beforeDir="false" afterPath="$PROJECT_DIR$/src/planner/specification_analyzer.py" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/src/rpc_server.py" beforeDir="false" afterPath="$PROJECT_DIR$/src/rpc_server.py" afterDir="false" />
       <change beforePath="$PROJECT_DIR$/src/utils/tosca.py" beforeDir="false" afterPath="$PROJECT_DIR$/src/utils/tosca.py" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/venv/lib/python3.6/site-packages/setuptools-40.8.0-py3.6.egg" beforeDir="false" afterPath="$PROJECT_DIR$/venv/lib/python3.6/site-packages/setuptools-40.8.0-py3.6.egg" afterDir="false" />
     </list>
     <option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
     <option name="SHOW_DIALOG" value="false" />
@@ -196,7 +198,14 @@
       <option name="project" value="LOCAL" />
       <updated>1571320780678</updated>
     </task>
-    <option name="localTasksCounter" value="12" />
+    <task id="LOCAL-00012" summary="fixed invalid type">
+      <created>1571326903674</created>
+      <option name="number" value="00012" />
+      <option name="presentableId" value="LOCAL-00012" />
+      <option name="project" value="LOCAL" />
+      <updated>1571326903675</updated>
+    </task>
+    <option name="localTasksCounter" value="13" />
     <servers />
   </component>
   <component name="Vcs.Log.Tabs.Properties">
@@ -223,7 +232,8 @@
     <MESSAGE value="slolved cap error" />
     <MESSAGE value="try to analyze policies" />
     <MESSAGE value="added libs in setup" />
-    <option name="LAST_COMMIT_MESSAGE" value="added libs in setup" />
+    <MESSAGE value="fixed invalid type" />
+    <option name="LAST_COMMIT_MESSAGE" value="fixed invalid type" />
   </component>
   <component name="XDebuggerManager">
     <breakpoint-manager>
@@ -233,16 +243,16 @@
           <line>64</line>
           <option name="timeStamp" value="1" />
         </line-breakpoint>
-        <line-breakpoint enabled="true" suspend="THREAD" type="python-line">
-          <url>file://$PROJECT_DIR$/src/planner/simple_spec_alayzer.py</url>
-          <line>39</line>
-          <option name="timeStamp" value="125" />
-        </line-breakpoint>
         <line-breakpoint enabled="true" suspend="THREAD" type="python-line">
           <url>file://$PROJECT_DIR$/src/utils/tosca.py</url>
-          <line>18</line>
+          <line>20</line>
           <option name="timeStamp" value="128" />
         </line-breakpoint>
+        <line-breakpoint enabled="true" suspend="THREAD" type="python-line">
+          <url>file://$PROJECT_DIR$/src/planner/planner.py</url>
+          <line>69</line>
+          <option name="timeStamp" value="211" />
+        </line-breakpoint>
       </breakpoints>
       <default-breakpoints>
         <breakpoint type="python-exception">

drip_planner2/src/planner/planner.py

@@ -29,7 +29,7 @@ class Planner:
 
     def add_required_nodes_to_template(self, required_nodes):
         for req_node in required_nodes:
-            node_template = tosca_util.node_type_2_node_template(req_node)
+            node_template = tosca_util.node_type_2_node_template(req_node, self.all_custom_def)
             self.tosca_template.nodetemplates.append(node_template)
         return self.tosca_template
 
@@ -55,75 +55,67 @@ class Planner:
                         return constraint[next(iter(constraint))]
             return None
 
-    # Resolve requirements. Go over all nodes and recursively resolve requirements till node has no requirements
-    # e.g. docker -> k8s -> cluster -> vm
     def resolve_requirements(self):
+        """ Resolve requirements. Go over all nodes and recursively resolve requirements till node has no
+        requirements  e.g. docker -> k8s -> cluster -> vm """
         for node in self.tosca_template.nodetemplates:
             logging.info('Resolving requirements for: ' + node.name)
             self.add_required_nodes(node)
         return self.required_nodes
 
     def add_required_nodes(self, node):
-        node_type = tosca_util.get_node_type_name(node)
-        missing_requirements = self.get_missing_requirements(node)
-        if not missing_requirements:
-            logging.debug('Node: ' + node_type + ' has no requirements')
+        """Adds the required nodes in self.required_nodes for an input node."""
+
+        # Get all requirements for node.
+        all_requirements = self.get_all_requirements(node)
+        if not all_requirements:
+            logging.debug('Node: ' + tosca_util.get_node_type_name(node) + ' has no requirements')
             return
-        matching_node = self.find_best_node(missing_requirements)
+        matching_node = self.find_best_node_for_requirements(all_requirements)
 
         # Only add node that is not in node_templates
         matching_node_type_name = next(iter(matching_node))
         matching_node_type_name = next(iter(matching_node))
-        matching_node_template = tosca_util.node_type_2_node_template(matching_node)
-        # For some reason the definitions of swarm are missing. We add them manually here
-        if len(matching_node_template.custom_def) < len(self.all_custom_def):
-            matching_node_template.custom_def.update(self.all_custom_def)
+        matching_node_template = tosca_util.node_type_2_node_template(matching_node, self.all_custom_def)
 
-        node = self.add_missing_requirements(node, missing_requirements, matching_node_template.name)
+        # Add the requirements to the node we analyzed. e.g. docker needed host now we added the type and name of host
+        node = self.add_requirements(node, all_requirements, matching_node_template.name)
         if not tosca_util.contains_node_type(self.required_nodes, matching_node_type_name):
             logging.info('  Adding: ' + str(matching_node_template.name))
             self.required_nodes.append(matching_node)
         # Find matching nodes for the new node's requirements
         self.add_required_nodes(matching_node)
 
-    def get_missing_requirements(self, node):
-        node_type_name = tosca_util.get_node_type_name(node)
-        node_requirements = tosca_util.get_node_requirements(node)
-        parent_type_requirements = tosca_util.get_parent_type_requirements(node, self.all_node_types)
+    def get_all_requirements(self, node):
+        """Returns  all requirements for an input node """
 
-        logging.debug('Looking for requirements for node: ' + node_type_name)
+        node_type_name = tosca_util.get_node_type_name(node)
+        logging.info('      Looking for requirements for node: ' + node_type_name)
+        # Get the requirements for this node from its definition e.g. docker: hostedOn k8s
         def_type = self.all_node_types[node_type_name]
-        def_requirements = []
-
+        all_requirements = []
         if 'requirements' in def_type.keys():
-            def_requirements = def_type['requirements']
-            logging.debug('Found requirements: ' + str(def_requirements) + ' for node: ' + node_type_name)
-
-        missing_requirements = []
-
-        if not node_requirements:
-            missing_requirements = def_requirements
-        elif def_requirements:
-            for def_requirement in def_requirements:
-                for key in def_requirement:
-                    for node_req in node_requirements:
-                        if key not in node_req:
-                            req_name = next(iter(def_requirement))
-                            def_requirement[req_name]['node'] = None
-                            missing_requirements.append(def_requirement)
-
-        # Make sure we have the definition. Otherwise we get an error in the recursion
-        # if 'derived_from' in def_type.keys() and not def_type['derived_from'] in custom_def.keys():
-        #     node.custom_def[def_type['derived_from']] = custom_def['derived_from']
+            all_requirements = def_type['requirements']
+            logging.info('      Found requirements: ' + str(all_requirements) + ' for node: ' + node_type_name)
+
+        # Get the requirements for this node from the template. e.g. wordpress: connectsTo mysql
+        # node_requirements =  tosca_util.get_node_requirements(node)
+        # if node_requirements:
+        #     all_requirements += node_requirements
+
+        # Put them all together
+        parent_requirements = tosca_util.get_ancestors_requirements(node, self.all_node_types, self.all_custom_def)
         parent_type = tosca_util.get_node_type_name(node)
-        if parent_type and parent_type_requirements:
-            logging.debug(
-                ' Adding to : ' + str(node_type_name) + '  parent requirements from: ' + str(parent_type))
-            missing_requirements = missing_requirements + parent_type_requirements
-            logging.debug('  missing_requirements: ' + str(missing_requirements))
-        return missing_requirements
+        if parent_type and parent_requirements:
+            logging.info(
+                '       Adding to : ' + str(node_type_name) + '  parent requirements from: ' + str(parent_type))
+            missing_requirements = all_requirements + parent_requirements
+            logging.debug('      all_requirements: ' + str(all_requirements))
+        return all_requirements
 
     def get_node_types_by_capability(self, cap):
+        """Returns  all nodes that have the  capability: cap and have interfaces. This way we  distinguish between
+        'abstract' and 'concrete' """
         candidate_nodes = {}
         for tosca_node_type in self.all_node_types:
             if tosca_node_type.startswith('tosca.nodes') and 'capabilities' in self.all_node_types[tosca_node_type]:
@@ -147,13 +139,15 @@ class Planner:
             capable_nodes[type_name] = candidate_nodes[type_name]
         return capable_nodes
 
-    def find_best_node(self, missing_requirements):
+    def find_best_node_for_requirements(self, all_requirements):
+        """Returns  the 'best' node for a set of requirements. Here we count the number of requiremets that the node
+        can cover and return the one which covers the most """
         matching_nodes = {}
         number_of_matching_requirement = {}
         # Loop requirements to find nodes per requirement
-        for req in missing_requirements:
-            for key in req:
-                capability = req[key]['capability']
+        for req in all_requirements:
+            if 'capability' in  req[next(iter(req))]:
+                capability = req[next(iter(req))]['capability']
                 logging.info('  Looking for nodes with capability: ' + capability)
                 # Find all nodes in the definitions that have the capability: capability
                 capable_nodes = self.get_node_types_by_capability(capability)
@@ -192,7 +186,8 @@ class Planner:
                     child_nodes[tosca_node_type] = self.all_node_types[tosca_node_type]
         return child_nodes
 
-    def add_missing_requirements(self, node, missing_requirements, capable_node_name):
+    def add_requirements(self, node, missing_requirements, capable_node_name):
+        """Add the requirements to the node """
         for req in missing_requirements:
             req[next(iter(req))]['node'] = capable_node_name
             if isinstance(node, NodeTemplate):

drip_planner2/src/planner/simple_spec_alayzer.py

@@ -37,14 +37,19 @@ class SimpleAnalyzer(SpecificationAnalyzer):
     def set_node_properties_for_policy(self, affected_node, policies):
         logging.info('Setting properties for: ' + str(affected_node.type))
 
-        ancestors_types = tosca_util.get_all_ancestors_types(affected_node, self.all_node_types)
+        ancestors_types = tosca_util.get_all_ancestors_types(affected_node, self.all_node_types, self.all_custom_def)
         if 'tosca.nodes.ARTICONF.Orchestrator' in ancestors_types:
             logging.info('Do Something')
-            if 'tosca.policies.ARTICONF.Performance.CPU' in policies:
-                logging.info('Do placement')
+            properties = tosca_util.get_all_ancestors_properties(affected_node, self.all_node_types,
+                                                                 self.all_custom_def)
+            for node_property in properties:
+                property_dict = {}
+                logging.info('property: ' + str(node_property.name) + ' val: ' + str(node_property.value))
+
         return affected_node
 
     def set_specs(self, node_name, policies, nodes_in_template):
+        logging.info('node_name: ' + str(node_name) + ' will implement policies: ' + str(len(policies)))
         for node in nodes_in_template:
             if node.name == node_name:
                 affected_node = node

drip_planner2/src/planner/specification_analyzer.py

@@ -21,7 +21,7 @@ class SpecificationAnalyzer(metaclass=ABCMeta):
         self.required_nodes = required_nodes
         self.nodes_in_template = []
         for req_node in required_nodes:
-            node_template = tosca_util.node_type_2_node_template(req_node)
+            node_template = tosca_util.node_type_2_node_template(req_node,self.all_custom_def)
             self.nodes_in_template.append(node_template)
 
         self.nodes_in_template += self.tosca_template.nodetemplates

drip_planner2/src/rpc_server.py

@@ -117,7 +117,7 @@ if __name__ == "__main__":
         test_planner_required_nodes = test_planner.set_infrastructure_specifications(test_planner_required_nodes)
         test_planner.add_required_nodes_to_template(test_planner_required_nodes)
         template = tosca_util.get_tosca_template_2_topology_template(test_planner.tosca_template)
-        # logger.info("template ----: \n" + template)
+        logger.info("template ----: \n" + template)
     else:
         logger.info("Input args: " + sys.argv[0] + ' ' + sys.argv[1] + ' ' + sys.argv[2])
         channel = init_chanel(sys.argv)

drip_planner2/src/utils/tosca.py

+import copy
 from itertools import chain
 
 from toscaparser.elements.nodetype import NodeType
@@ -11,6 +12,7 @@ node_type_key_names_to_remove = ['capabilities', 'derived_from']
 
 
 def get_node_type_name(node):
+    """Returns  the requirements for an input node as described in the template not in the node's definition """
     if isinstance(node, NodeTemplate):
         if node.type:
             if node.type and isinstance(node.type, str):
@@ -53,17 +55,27 @@ def get_node_type_requirements(type_name, all_nodes):
     return None
 
 
-def get_parent_type_requirements(node, all_nodes):
+def get_ancestors_requirements(node, all_nodes, all_custom_def, parent_requirements=None):
+    """Recursively get all requirements all the way to the ROOT including the input node's"""
+    if not parent_requirements:
+        parent_requirements = []
     if isinstance(node, NodeTemplate):
-
+        # If node has parent and parent has requirements add them
         if node.parent_type and node.parent_type.requirements:
-            parent_type_requirements = node.parent_type.requirements
-        else:
-            parent_type_requirements = {}
+            if isinstance(node.parent_type.requirements, dict):
+                parent_requirements.append(node.parent_type.requirements)
+            elif isinstance(node.parent_type.requirements, list):
+                parent_requirements.extend(node.parent_type.requirements)
+            # Make parent type to NodeTemplate to continue
+            if node.parent_type.type:
+                parent_template = node_type_2_node_template({'name': all_nodes[node.parent_type.type]}, all_custom_def)
+            if parent_template:
+                get_ancestors_requirements(parent_template, all_nodes, parent_requirements)
     elif isinstance(node, dict):
         node_type_name = get_node_type_name(node)
-        parent_type_requirements = get_node_type_requirements(node_type_name, all_nodes)
-    return parent_type_requirements
+        node_template = node_type_2_node_template({'name': all_nodes[node_type_name]}, all_custom_def)
+        get_ancestors_requirements(node_template, all_nodes, all_custom_def, parent_requirements)
+    return parent_requirements
 
 
 def get_node_types_with_interface(nodes):
@@ -74,7 +86,7 @@ def get_node_types_with_interface(nodes):
     return node_types_with_interface
 
 
-def node_type_2_node_template(node_type):
+def node_type_2_node_template(node_type, all_custom_def):
     node_template_dict = {}
     type_name = next(iter(node_type))
     node_type_array = type_name.split(".")
@@ -88,7 +100,19 @@ def node_type_2_node_template(node_type):
 
     if 'type' in node_type[next(iter(node_type))]:
         node_type[next(iter(node_type))].pop('type')
-    return NodeTemplate(name, node_template_dict, node_type)
+
+    node_template = NodeTemplate(name, node_template_dict, node_type)
+    # For some reason the tosca.nodes.ARTICONF.Orchestrator doesn't  have all definitions so we need to add them
+    # manually. We get 'toscaparser.common.exception.InvalidTypeError: Type "tosca.nodes.ARTICONF.Orchestrator"
+    # is not a valid type.'
+    if len(node_template.custom_def) < len(all_custom_def):
+        for def_key in all_custom_def:
+            if isinstance(def_key, dict):
+                node_template.custom_def.update(def_key)
+            else:
+                node_template.custom_def[def_key] = all_custom_def[def_key]
+
+    return node_template
 
 
 def get_tosca_template_2_topology_template(template):
@@ -135,12 +159,32 @@ def get_node_by_type(node_type, all_nodes):
     return all_nodes[node_type]
 
 
-def get_all_ancestors_types(child_node, all_nodes, ancestors_types=None):
+def get_all_ancestors_types(child_node, all_nodes, all_custom_def, ancestors_types=None):
     if not ancestors_types:
         ancestors_types = [get_node_type_name(child_node)]
     parent_type = get_parent_type(child_node)
     if parent_type:
         ancestors_types.append(parent_type)
-        parent_type = node_type_2_node_template({'name': all_nodes[parent_type]})
-        get_all_ancestors_types(parent_type, all_nodes, ancestors_types)
+        parent_type = node_type_2_node_template({'name': all_nodes[parent_type]}, all_custom_def)
+        get_all_ancestors_types(parent_type, all_nodes, all_custom_def,ancestors_types)
     return ancestors_types
+
+
+def get_all_ancestors_properties(node, all_nodes, all_custom_def, ancestors_properties=None, ancestors_types=None):
+    if not ancestors_properties:
+        ancestors_properties = []
+        ancestors_properties_names = []
+        if node.get_properties_objects():
+            ancestors_properties.extend(node.get_properties_objects())
+    if not ancestors_types:
+        ancestors_types = get_all_ancestors_types(node, all_nodes,all_custom_def)
+
+    for ancestors_type in ancestors_types:
+        ancestor = node_type_2_node_template({'name': all_nodes[ancestors_type]}, all_custom_def)
+        if ancestor.get_properties_objects():
+            for ancestor_prop in ancestor.get_properties_objects():
+                if ancestor_prop.name not in ancestors_properties_names:
+                    ancestors_properties_names.append(ancestor_prop.name)
+                    ancestors_properties.append(ancestor_prop)
+
+    return ancestors_properties