Refactored part of Connected/Similarity code

src/data-hub/role-stage-discovery-microservice/app/db/entities/connected_node.py

 
 class NodeC:
     
-    def __init__(self, cluster_label, node_layer, finished_time, latitude_Destination, longitude_Destination, travelID, travelPrice, uniqueID, userID):
+    def __init__(self, cluster_label, node_layer, uniqueID):
         self.cluster_label = cluster_label
         self.node_layer = node_layer
-        self.finished_time = finished_time
-        self.latitude_Destination = latitude_Destination
-        self.longitude_Destination = longitude_Destination
-        self.travelID = travelID
-        self.travelPrice = travelPrice
         self.uniqueID = uniqueID        
\ No newline at end of file
-        self.userID = userID
\ No newline at end of file

src/data-hub/role-stage-discovery-microservice/app/db/repository.py

@@ -90,7 +90,7 @@ class Repository(MongoRepositoryBase):
         result = super().insert_many(self._connected_clusters_collection, clusterDictArray)
         return result
         
-    def get_connected_clusters(self, run_id=None):#, layer_name: str):
+    def get_connected_clusters(self, run_id: str=None):#, layer_name: str):
         ''' Get Connected Clusters Data from DB '''
         if (run_id == None):
             entries = super().get_entries(self._connected_clusters_collection, projection={'_id': 0})
@@ -115,8 +115,7 @@ class Repository(MongoRepositoryBase):
         #super().insert_entry(self._connected_clusters_collection, outputJSON)
         return result
     
-    #TODO
-    def get_similarity(self, run_id=None):
+    def get_similarity(self, run_id: str=None):
         ''' Get Similarity Data from DB '''
         if (run_id == None):
             entries = super().get_entries(self._similarity_collection, projection={'_id': 0})
@@ -138,12 +137,12 @@ class Repository(MongoRepositoryBase):
         result = super().insert_entry(self._connected_run, conRunTimestamp)
         return result
 
-    def get_connected_run(self, run_id= None):
+    def get_connected_run(self, run_id: str= None):
         ''' Get Connected Run Data from DB '''
         if (run_id == None):
             entries = super().get_entries(self._connected_run)
         else:
-            entries = super().get_entries(self._connected_run, selection={'_id' : run_id})
+            entries = super().get_entries(self._connected_run, selection={'_id' : run_id}, projection={'_id': 1, 'Datetime': 1})
 
         output = []
         for e in entries:

src/data-hub/role-stage-discovery-microservice/app/processing/similarityFiles/calculateSimilarity.py

@@ -30,7 +30,6 @@ def minMaxFunction(iIndex,jIndex,clusterList) -> Dict[str,int]:
     for curCluster in clusterList:      #jCluster.cluster_layer == iCluster.cluster_layer, so i only compare to one
 
         curLayer = curCluster.cluster_layer
-        curLabel = curCluster.cluster_label
         if(( curLayer != iCluster.cluster_layer) 
         and ( curCluster.cluster_connClustDict.__contains__(iCluster.cluster_label)) 
         and ( curCluster.cluster_connClustDict.__contains__(jCluster.cluster_label))):
@@ -71,7 +70,6 @@ def calcEuclideanDist(iIndex,jIndex,clusterList) -> Dict[str,float]:
     for curCluster in clusterList:      #jCluster.cluster_layer == iCluster.cluster_layer, so i only compare to one
         
         curLayer = curCluster.cluster_layer
-        curLabel = curCluster.cluster_label #debugOnly
 
         #considering only clusters from other layers for distance calc
         if( curLayer != iCluster.cluster_layer):
@@ -116,7 +114,7 @@ def calculateSimilarity(inputLayerDict):
 
     ''' Calculates the similarity between clusters contained in the "inputLayerDict". Similarity is calculated for each combination of 2 clusters from the SAME layer.
 
-    :param Dict{layername: Layer} inputLayerDict: Contains the associated Layer and Clusters objects. The dictonary KEY is layername, the Value is a Layer Object. The Layer object has an attribute cluster_Dict which stores the clusters in the Layer.
+    :param Dict{layername: LayerC} inputLayerDict: Contains the associated Layer and Clusters objects. The dictonary KEY is layername, the Value is a LayerC Object. The LayerC object has an attribute cluster_Dict which stores the clusters in the LayerC.
 
 
     :returns: Dict{tuple(cluster_label1, cluster_label2) : Dict{layername, similarityValue}}. Returns a Dictionary with a tuple of 2 clusters as KEY, and a Dictionary with the computed similarity of the clusters in regard to each layer as VALUE
@@ -129,8 +127,8 @@ def calculateSimilarity(inputLayerDict):
     clusterList = list()
     
 
-    for curLayer in inputLayerDict.values():        
-        for curCluster in curLayer.cluster_Dict.values():
+    for curLayerC in inputLayerDict.values():        
+        for curCluster in curLayerC.cluster_Dict.values():
             clusterList.append(curCluster)
     #print(" Nr. of clusters: "+str(len(clusterList)))
     

src/data-hub/role-stage-discovery-microservice/app/processing/similarityFiles/calculateWeights.py

@@ -15,12 +15,12 @@ def calculateWeights(inputLayerDict) -> Dict[str,LayerC]:
 
     ''' Calculates the nr of connections/weights between the clusters contained in the "inputLayerDict". Connections are made between clusters from DIFFERENT layers.
 
-    :param Dict{string: Layer} inputLayerDict: Contains the associated Layer and Clusters objects. The dictonary KEY is layername, the Value is a Layer Object. The Layer object has an attribute cluster_Dict which stores the clusters in the Layer.
+    :param Dict{string: LayerC} inputLayerDict: Contains the associated LayerC and ConnClusters objects. The dictonary KEY is layername, the Value is a Layer Object. The Layer object has an attribute cluster_Dict which stores the clusters in the LayerC.
 
 
-    :returns: Dict{layername: Layer}. Returns the inputLayerDict with the added connections in the attributes cluster_connClustDict and cluster_connNodesDict
+    :returns: Dict{layername: LayerC}. Returns the inputLayerDict with the added connections in the attributes cluster_connClustDict and cluster_connNodesDict
     
-    :rtype: Dict{string: Layer}
+    :rtype: Dict{string: LayerC}
     '''    
 
 
@@ -52,13 +52,15 @@ def calculateWeights(inputLayerDict) -> Dict[str,LayerC]:
         while ( j<len(nodeList)):
             jNode = nodeList[j]
             #if there is a connection
-            #print("\n ### \n"+iNode.uniqueID +" "+ iNode.node_layer +"\n"+ jNode.uniqueID +" "+ jNode.node_layer )
+            #Compute a connection 
 
             if (iNode.node_layer != jNode.node_layer) and (iNode.uniqueID == jNode.uniqueID):
                 iOldTuple = (iNode.uniqueID,iNode.cluster_label)
                 jOldTuple= (jNode.uniqueID,jNode.cluster_label)
                 iOldKey = frozenset(iOldTuple)
                 jOldKey = frozenset(jOldTuple)
+                #iForeignKey =
+                #jForeignKey =
                 #Check if old node dicts has this node: if not add to ConnDictionary and to OldNodesDict
                 #              Layer                .             Cluster             .    OldNodesDict    .    Does not contain the OTHER node
                 
@@ -86,11 +88,6 @@ def calculateWeights(inputLayerDict) -> Dict[str,LayerC]:
             j+=1   
         i+=1
 
-    #deleting cluster_containedNodesDicts/// No longer needed
-    #for curLayer in inputLayerDict.values():
-     #   for curCluster in curLayer.cluster_Dict.values():
-     #       inputLayerDict[curCluster.cluster_layer].cluster_Dict[curCluster.cluster_label].cluster_containedNodesDict = dict()
-
     print("Finished calculateWeights")
 
     #store weights in database?

src/data-hub/role-stage-discovery-microservice/app/processing/similarityFiles/dataInput.py

@@ -10,9 +10,10 @@ import json
 import requests
 from routes.clustersets import get_by_name
 
-#TEST ONLY
+#Only used for testing
 from routes.connClusters import get_conn_clusters
 from routes.similarity import get_similarity
+from routes.connRun import get_connected_run
 
 
 def getClusterDataFromSwagger(limitNrCluster,limitNrNodes):
@@ -23,19 +24,12 @@ def getClusterDataFromSwagger(limitNrCluster,limitNrNodes):
 
     :param int limitNrNodes: Limits Clusters considered. None or <0 values == No limit
 
-    :returns: Dict{layername: Layer}. Returns a Dict with the data gathered from the DB
+    :returns: Dict{layername: LayerC}. Returns a Dict with the data gathered from the DB
     
-    :rtype: Dict{string: Layer}
+    :rtype: Dict{string: LayerC}
     '''    
 
     print("Entered dataInput")
-    # ??? OBSOLETE ???
-    # oldBigTestClusters https://drive.google.com/uc?export=download&id=1l4gHBwrG_N4pCL5-MfWJk2szNrF3VnpG  it takes a whie to download
-    # smallTestClusters https://drive.google.com/uc?export=download&id=1cMoGtmi-XouSDM9DRl-ddmPkf2Bm7sk7
-    # smallTestOnlyLocationClusters https://drive.google.com/uc?export=download&id=1wBT9vi7aS4rE4qOWHEyLEfQ2KbmpBK9e
-    # smallTestOnlyPriceClusters https://drive.google.com/uc?export=download&id=1g9pEOOpDMBj6yZOlFj7HfOoMOAFTvPOW
-    # smallTestOnlyTimeClusters https://drive.google.com/uc?export=download&id=1XKXQHEC5ubJHmntQBNnzgfpEZl6OXE_B
-
     listURLs = []    
     #"""
     listURLs.append('http://articonf1.itec.aau.at:30103/api/layers/Price_Layer/clusters')
@@ -78,9 +72,9 @@ def getClusterDataFromMongo(layerNameList,limitNrCluster,limitNrNodes):
 
     :param int limitNrNodes: Limits Clusters considered. None or <0 values == No limit
 
-    :returns: Dict{layername: Layer}. Returns a Dict with the data gathered from the DB
+    :returns: Dict{layername: LayerC}. Returns a Dict with the data gathered from the DB
     
-    :rtype: Dict{string: Layer}
+    :rtype: Dict{string: LayerC}
     '''    
 
     layerDict = dict()
@@ -102,77 +96,6 @@ def getClusterDataFromMongo(layerNameList,limitNrCluster,limitNrNodes):
     return layerDict
 
 
-def populateWithNewNodesSingleLayer(inputData, layerDict) -> Dict[str,LayerC]:
-    ''' Gets Layer,Cluster and Node data from a JSON format, and appends it into the "layerDict" dictionary. A single Layer only
-
-    :param inputData: JSON data to be formated.
-
-    :param Dict{string: Layer} layerDict: If it's empty/null a new one will be created. Otherwise contains the associated Layer and Clusters objects to which date is appended. The dictonary KEY is layername, the Value is a Layer Object. The Layer object has an attribute cluster_Dict which stores the clusters in the Layer.
-
-    :returns: Dict{layername: Layer}. Returns the inputLayerDict with the added data from the JSON
-    
-    :rtype: Dict{string: Layer}
-    '''  
-    print("Entered populateWithNewNodes")
-    if(layerDict == None):
-        layerDict = dict()
-    print(" Layer: "+inputData[0].get("layer_name"))
-
-    curLayerName = None
-    #newClusterDict
-    #clusterDict = layerDict.get(curCluster.get("layer_name"),dict())
-    for curCluster in inputData:
-        if(curCluster.get("layer_name")!= curLayerName):
-            clusterDict = layerDict.get(curCluster.get("layer_name"),dict())
-            curLayerName = curCluster.get("layer_name")
-            
-        oldCluster = clusterDict.get(curCluster.get("cluster_label"),None)
-        if oldCluster is None: #means this is a new cluster
-            cluster_containedNodesDict = dict()
-        else: #means this is an already existing cluster
-            cluster_containedNodesDict = oldCluster.cluster_containedNodesDict
-
-        for curNode in curCluster.get("nodes"):
-            #totalNodesCount+=1
-            newNode = NodeC(
-                curCluster.get("cluster_label"),
-                curLayerName,
-                curNode.get("Finished_time"),
-                curNode.get("Latitude_Destination"),
-                curNode.get("Longitude_Destination"),
-                curNode.get("TravelID"),
-                curNode.get("TravelPrice"),
-                curNode.get("UniqueID"),
-                curNode.get("UserID"))
-            if(newNode != None):
-                if(newNode.uniqueID!= None and newNode.cluster_label!= None and newNode.node_layer!= None):
-                    auxtuple = (newNode.uniqueID,newNode.cluster_label)
-                    key = frozenset(auxtuple)
-                    cluster_containedNodesDict[key]= newNode #overwrite if already there
-
-        #finished node
-
-        if oldCluster is None:
-            #    def __init__(self,cluster_label,cluster_layer,cluster_containedNodesDict,cluster_connNodesDict, cluster_connectionsNr,cluster_connClustDict):
-            newCluster = ClusterC(
-                curCluster.get("cluster_label"),
-                curLayerName,
-                None,
-                cluster_containedNodesDict,
-                dict(),         #will populate the dict fields later later
-                dict()) #may not be empty anymore) 
-
-            clusterDict[newCluster.cluster_label] = newCluster
-        else:
-            oldCluster.cluster_containedNodesDict = cluster_containedNodesDict
-            clusterDict[curCluster.get("cluster_label")] = oldCluster               
-        #finished cluster
-        newLayer = LayerC(curLayerName,clusterDict)
-        layerDict[curLayerName]= newLayer
-
-    ##########TESTEAAAZAA
-    return layerDict
-
 def populateWithNewNodesSingleLayer(inputData, layerDict, limitNrNodes) -> Dict[str,LayerC]:
     ''' Gets Layer,Cluster and Node data from a JSON format, and appends it into the "layerDict" dictionary.
 
@@ -206,24 +129,18 @@ def populateWithNewNodesSingleLayer(inputData, layerDict, limitNrNodes) -> Dict[
         else: #means this is an already existing cluster
             cluster_containedNodesDict = oldCluster.cluster_containedNodesDict
         
-        
         for curNode in curCluster.get("nodes"):
             #totalNodesCount+=1
             newNode = NodeC(
                 curCluster.get("cluster_label"),
                 curLayerName,
-                curNode.get("Finished_time"),
-                curNode.get("Latitude_Destination"),
-                curNode.get("Longitude_Destination"),
-                curNode.get("TravelID"),
-                curNode.get("TravelPrice"),
-                curNode.get("UniqueID"),
-                curNode.get("UserID"))
+                curNode.get("UniqueID"))
             if(newNode != None):
                 if(newNode.uniqueID!= None and newNode.cluster_label!= None and newNode.node_layer!= None):
                     if( limitNrNodes>0):
                         auxtuple = (newNode.uniqueID,newNode.cluster_label)
                         key = frozenset(auxtuple)
+                        #check for duplicates #TODO
                         cluster_containedNodesDict[key]= newNode #overwrite if already there
                         limitNrNodes-=1
 
@@ -249,82 +166,6 @@ def populateWithNewNodesSingleLayer(inputData, layerDict, limitNrNodes) -> Dict[
 
     return layerDict
 
-#deprecated
-def populateWithNewNodesAllLayers(inputData,layerDict) -> Dict[str,LayerC]:
-    ''' Gets Layer,Cluster and Node data from a JSON format, and appends it into the "layerDict" dictionary.
-
-    :param inputData: JSON data to be formated.
-
-    :param Dict{string: Layer} layerDict: If it's empty/null a new one will be created. Otherwise contains the associated Layer and Clusters objects to which date is appended. The dictonary KEY is layername, the Value is a Layer Object. The Layer object has an attribute cluster_Dict which stores the clusters in the Layer.
-
-    :returns: Dict{layername: Layer}. Returns the inputLayerDict with the added data from the JSON
-
-    :rtype: Dict{string: Layer}
-    '''  
-    
-    print("Entered populateWithNewNodes")
-    if(layerDict == None):
-        layerDict = dict()
-
-    for curLayer in inputData:
-        
-        clusterDict = layerDict.get(curLayer.get("layer_name"),dict()) #gets the Old dict or an empty if none is found i.e for a new layer
-        curLayerName = curLayer.get("layer_name")
-        for curCluster in curLayer.get("clusters"):         
-            oldCluster = clusterDict.get(curCluster.get("cluster_label"),None)
-
-
-            if oldCluster is None: #means this is a new cluster
-                cluster_containedNodesDict = dict()
-            else: #means this is an already existing cluster
-                cluster_containedNodesDict = oldCluster.cluster_containedNodesDict
-            
-
-            #SORT NODES?  
-            for curNode in curCluster.get("nodes"):
-                if(curNode != None):
-                    if(curNode.uniqueID != None):
-                        newNode = NodeC(
-                            curCluster.get("cluster_label"),
-                            curLayerName,
-                            curNode.get("Finished_time"),
-                            curNode.get("Latitude_Destination"),
-                            curNode.get("Longitude_Destination"),
-                            curNode.get("TravelID"),
-                            curNode.get("TravelPrice"),
-                            curNode.get("UniqueID"),
-                            curNode.get("UserID"))
-
-                        auxtuple = (newNode.uniqueID,newNode.cluster_label)
-                        key = frozenset(auxtuple)
-                        cluster_containedNodesDict[key]= newNode #overwrite if already there
-
-            
-            #finished node
-
-            if oldCluster is None:
-                newCluster = ClusterC(
-                    curCluster.get("cluster_label"),
-                    curLayer.get("layer_name"),
-                    None,
-                    cluster_containedNodesDict,
-                    dict(),
-                    dict()) #may not be empty anymore) 
-
-                clusterDict[newCluster.cluster_label] = newCluster
-            else:
-                #only cluster_containedNodesDict should change
-                oldCluster.cluster_containedNodesDict = cluster_containedNodesDict
-                #cluster_connNr and clusterConnDict should stay the same
-                clusterDict[curCluster.get("cluster_label")] = oldCluster               
-        #finished cluster
-        newLayer = LayerC(curLayer.get("layer_name"),clusterDict)
-        layerDict[curLayer.get("layer_name")]= newLayer
-    #finished layer
-    print("Finished populateWithNewNodes")
-    return layerDict
-
-
 
 def getConnClusterDataFromMongo():
     mongoArray = get_conn_clusters()
@@ -332,12 +173,15 @@ def getConnClusterDataFromMongo():
 
     return outputDict
 
-def getSimilarityDataFromMOngo():
+def getSimilarityDataFromMongo():
     result = get_similarity()
     return result
 
+def getConnectedRunDataFromMongo():
+    result = get_connected_run()
+    return result
+
 def convertRetrievedClustersFromMongo(inputArray):
-    ####TODO#### Not tested thoroughly
     
     LayerDict = {}
     for entry in inputArray:

src/data-hub/role-stage-discovery-microservice/app/processing/similarityFiles/dataOutput.py

@@ -2,10 +2,9 @@
 import json
 import requests
 import datetime
-from routes.connClusters import add_conn_clusters
-from routes.similarity import add_similarity
-from routes.connRun import add_connected_run
 from processing.similarityFiles.miscFunctions import *
+from db.repository import Repository
+repo = Repository()
 
 def outputFileLayerFunction(layerDict,limitNrNodes,limitNrCluster,runId):
     ''' Writes the layerDict data to a JSON file.
@@ -101,3 +100,42 @@ def outputMongoSimilarity(inputDict,runId):
 
     ''' 
     add_similarity(inputDict,runId)
+
+def add_connected_run():
+
+    '''
+        Inserts Run with current Time into the DB
+        
+        :returns: Returns the _id of the connected_run entry in the DB
+    
+        :rtype: string
+    '''    
+
+    currentTime = datetime.datetime.now()
+    runDict = {"Datetime" : str(currentTime)}
+    inserted_result = repo.add_connected_run(runDict)
+    return str(inserted_result.inserted_id)
+
+def add_conn_clusters(inputDict,runId):
+    ''' Stores connected_clusters in the database.
+
+    :param Dict() inputDict: Contains the data to insert
+
+    :param string runId: Id of the Run 
+
+    ''' 
+    
+    outputJSON = convertLayerDictToJSON(inputDict,runId)
+    repo.add_connected_clusters(outputJSON)
+
+def add_similarity(inputDict,runId):
+    ''' Stores cluster_similarity in the database.
+
+    :param Dict() inputDict: Contains the data to insert
+
+    :param string runId: Id of the Run 
+
+    ''' 
+
+    outputJSON = convertSimilarityDictToJSON(inputDict,runId)
+    repo.add_similarity(outputJSON)
\ No newline at end of file

src/data-hub/role-stage-discovery-microservice/app/resultLayerDictN2999C60.json

@@ -2,7 +2,7 @@
     {
         "cluster_label": 0,
         "cluster_layer": "Price_Layer",
-        "cluster_runId": "5f02e43b53a73a48d0eaaed5",
+        "cluster_runId": "5f033232366be85ec1afca7b",
         "cluster_connClustDict": {
             "-1": 42,
             "0": 21
@@ -1267,7 +1267,7 @@
     {
         "cluster_label": 6,
         "cluster_layer": "Price_Layer",
-        "cluster_runId": "5f02e43b53a73a48d0eaaed5",
+        "cluster_runId": "5f033232366be85ec1afca7b",
         "cluster_connClustDict": {},
         "cluster_connNodesDict": [],
         "cluster_containedNodesDict": []
@@ -1275,7 +1275,7 @@
     {
         "cluster_label": 17,
         "cluster_layer": "Price_Layer",
-        "cluster_runId": "5f02e43b53a73a48d0eaaed5",
+        "cluster_runId": "5f033232366be85ec1afca7b",
         "cluster_connClustDict": {},
         "cluster_connNodesDict": [],
         "cluster_containedNodesDict": []
@@ -1283,7 +1283,7 @@
     {
         "cluster_label": 18,
         "cluster_layer": "Price_Layer",
-        "cluster_runId": "5f02e43b53a73a48d0eaaed5",
+        "cluster_runId": "5f033232366be85ec1afca7b",
         "cluster_connClustDict": {},
         "cluster_connNodesDict": [],
         "cluster_containedNodesDict": []
@@ -1291,7 +1291,7 @@
     {
         "cluster_label": 7,
         "cluster_layer": "Price_Layer",
-        "cluster_runId": "5f02e43b53a73a48d0eaaed5",
+        "cluster_runId": "5f033232366be85ec1afca7b",
         "cluster_connClustDict": {},
         "cluster_connNodesDict": [],
         "cluster_containedNodesDict": []
@@ -1299,7 +1299,7 @@
     {
         "cluster_label": 2,
         "cluster_layer": "Price_Layer",
-        "cluster_runId": "5f02e43b53a73a48d0eaaed5",
+        "cluster_runId": "5f033232366be85ec1afca7b",
         "cluster_connClustDict": {},
         "cluster_connNodesDict": [],
         "cluster_containedNodesDict": []
@@ -1307,7 +1307,7 @@
     {
         "cluster_label": 1,
         "cluster_layer": "Price_Layer",
-        "cluster_runId": "5f02e43b53a73a48d0eaaed5",
+        "cluster_runId": "5f033232366be85ec1afca7b",
         "cluster_connClustDict": {},
         "cluster_connNodesDict": [],
         "cluster_containedNodesDict": []
@@ -1315,7 +1315,7 @@
     {
         "cluster_label": 20,
         "cluster_layer": "Price_Layer",
-        "cluster_runId": "5f02e43b53a73a48d0eaaed5",
+        "cluster_runId": "5f033232366be85ec1afca7b",
         "cluster_connClustDict": {},
         "cluster_connNodesDict": [],
         "cluster_containedNodesDict": []
@@ -1323,7 +1323,7 @@
     {
         "cluster_label": 5,
         "cluster_layer": "Price_Layer",
-        "cluster_runId": "5f02e43b53a73a48d0eaaed5",
+        "cluster_runId": "5f033232366be85ec1afca7b",
         "cluster_connClustDict": {},
         "cluster_connNodesDict": [],
         "cluster_containedNodesDict": []
@@ -1331,7 +1331,7 @@
     {
         "cluster_label": 4,
         "cluster_layer": "Price_Layer",
-        "cluster_runId": "5f02e43b53a73a48d0eaaed5",
+        "cluster_runId": "5f033232366be85ec1afca7b",
         "cluster_connClustDict": {},
         "cluster_connNodesDict": [],
         "cluster_containedNodesDict": []
@@ -1339,7 +1339,7 @@
     {
         "cluster_label": 15,
         "cluster_layer": "Price_Layer",
-        "cluster_runId": "5f02e43b53a73a48d0eaaed5",
+        "cluster_runId": "5f033232366be85ec1afca7b",
         "cluster_connClustDict": {},
         "cluster_connNodesDict": [],
         "cluster_containedNodesDict": []
@@ -1347,7 +1347,7 @@
     {
         "cluster_label": 11,
         "cluster_layer": "Price_Layer",
-        "cluster_runId": "5f02e43b53a73a48d0eaaed5",
+        "cluster_runId": "5f033232366be85ec1afca7b",
         "cluster_connClustDict": {},
         "cluster_connNodesDict": [],
         "cluster_containedNodesDict": []
@@ -1355,7 +1355,7 @@
     {
         "cluster_label": 19,
         "cluster_layer": "Price_Layer",
-        "cluster_runId": "5f02e43b53a73a48d0eaaed5",
+        "cluster_runId": "5f033232366be85ec1afca7b",
         "cluster_connClustDict": {},
         "cluster_connNodesDict": [],
         "cluster_containedNodesDict": []
@@ -1363,7 +1363,7 @@
     {
         "cluster_label": 16,
         "cluster_layer": "Price_Layer",
-        "cluster_runId": "5f02e43b53a73a48d0eaaed5",
+        "cluster_runId": "5f033232366be85ec1afca7b",
         "cluster_connClustDict": {},
         "cluster_connNodesDict": [],
         "cluster_containedNodesDict": []
@@ -1371,7 +1371,7 @@
     {
         "cluster_label": 3,
         "cluster_layer": "Price_Layer",
-        "cluster_runId": "5f02e43b53a73a48d0eaaed5",
+        "cluster_runId": "5f033232366be85ec1afca7b",
         "cluster_connClustDict": {},
         "cluster_connNodesDict": [],
         "cluster_containedNodesDict": []
@@ -1379,7 +1379,7 @@
     {
         "cluster_label": 9,
         "cluster_layer": "Price_Layer",
-        "cluster_runId": "5f02e43b53a73a48d0eaaed5",
+        "cluster_runId": "5f033232366be85ec1afca7b",
         "cluster_connClustDict": {},
         "cluster_connNodesDict": [],
         "cluster_containedNodesDict": []
@@ -1387,7 +1387,7 @@
     {
         "cluster_label": 8,
         "cluster_layer": "Price_Layer",
-        "cluster_runId": "5f02e43b53a73a48d0eaaed5",
+        "cluster_runId": "5f033232366be85ec1afca7b",
         "cluster_connClustDict": {},
         "cluster_connNodesDict": [],
         "cluster_containedNodesDict": []
@@ -1395,7 +1395,7 @@
     {
         "cluster_label": 10,
         "cluster_layer": "Price_Layer",
-        "cluster_runId": "5f02e43b53a73a48d0eaaed5",
+        "cluster_runId": "5f033232366be85ec1afca7b",
         "cluster_connClustDict": {},
         "cluster_connNodesDict": [],
         "cluster_containedNodesDict": []
@@ -1403,7 +1403,7 @@
     {
         "cluster_label": 13,
         "cluster_layer": "Price_Layer",
-        "cluster_runId": "5f02e43b53a73a48d0eaaed5",
+        "cluster_runId": "5f033232366be85ec1afca7b",
         "cluster_connClustDict": {},
         "cluster_connNodesDict": [],
         "cluster_containedNodesDict": []
@@ -1411,7 +1411,7 @@
     {
         "cluster_label": 14,
         "cluster_layer": "Price_Layer",
-        "cluster_runId": "5f02e43b53a73a48d0eaaed5",
+        "cluster_runId": "5f033232366be85ec1afca7b",
         "cluster_connClustDict": {},
         "cluster_connNodesDict": [],
         "cluster_containedNodesDict": []
@@ -1419,7 +1419,7 @@
     {
         "cluster_label": -1,
         "cluster_layer": "FinishedTime_Layer",
-        "cluster_runId": "5f02e43b53a73a48d0eaaed5",
+        "cluster_runId": "5f033232366be85ec1afca7b",
         "cluster_connClustDict": {
             "0": 98
         },
@@ -2822,7 +2822,7 @@
     {
         "cluster_label": 2564,
         "cluster_layer": "FinishedTime_Layer",
-        "cluster_runId": "5f02e43b53a73a48d0eaaed5",
+        "cluster_runId": "5f033232366be85ec1afca7b",
         "cluster_connClustDict": {},
         "cluster_connNodesDict": [],
         "cluster_containedNodesDict": []
@@ -2830,7 +2830,7 @@
     {
         "cluster_label": 729,
         "cluster_layer": "FinishedTime_Layer",
-        "cluster_runId": "5f02e43b53a73a48d0eaaed5",
+        "cluster_runId": "5f033232366be85ec1afca7b",
         "cluster_connClustDict": {},
         "cluster_connNodesDict": [],
         "cluster_containedNodesDict": []
@@ -2838,7 +2838,7 @@
     {
         "cluster_label": 772,
         "cluster_layer": "FinishedTime_Layer",
-        "cluster_runId": "5f02e43b53a73a48d0eaaed5",
+        "cluster_runId": "5f033232366be85ec1afca7b",
         "cluster_connClustDict": {},
         "cluster_connNodesDict": [],
         "cluster_containedNodesDict": []
@@ -2846,7 +2846,7 @@
     {
         "cluster_label": 2756,
         "cluster_layer": "FinishedTime_Layer",
-        "cluster_runId": "5f02e43b53a73a48d0eaaed5",
+        "cluster_runId": "5f033232366be85ec1afca7b",
         "cluster_connClustDict": {},
         "cluster_connNodesDict": [],
         "cluster_containedNodesDict": []
@@ -2854,7 +2854,7 @@
     {
         "cluster_label": 3213,
         "cluster_layer": "FinishedTime_Layer",
-        "cluster_runId": "5f02e43b53a73a48d0eaaed5",
+        "cluster_runId": "5f033232366be85ec1afca7b",
         "cluster_connClustDict": {},
         "cluster_connNodesDict": [],
         "cluster_containedNodesDict": []
@@ -2862,7 +2862,7 @@
     {
         "cluster_label": 2362,
         "cluster_layer": "FinishedTime_Layer",
-        "cluster_runId": "5f02e43b53a73a48d0eaaed5",
+        "cluster_runId": "5f033232366be85ec1afca7b",
         "cluster_connClustDict": {},
         "cluster_connNodesDict": [],
         "cluster_containedNodesDict": []
@@ -2870,7 +2870,7 @@
     {
         "cluster_label": 4777,
         "cluster_layer": "FinishedTime_Layer",
-        "cluster_runId": "5f02e43b53a73a48d0eaaed5",
+        "cluster_runId": "5f033232366be85ec1afca7b",
         "cluster_connClustDict": {},
         "cluster_connNodesDict": [],
         "cluster_containedNodesDict": []
@@ -2878,7 +2878,7 @@
     {
         "cluster_label": 2745,
         "cluster_layer": "FinishedTime_Layer",
-        "cluster_runId": "5f02e43b53a73a48d0eaaed5",
+        "cluster_runId": "5f033232366be85ec1afca7b",
         "cluster_connClustDict": {},
         "cluster_connNodesDict": [],
         "cluster_containedNodesDict": []
@@ -2886,7 +2886,7 @@
     {
         "cluster_label": 5375,
         "cluster_layer": "FinishedTime_Layer",
-        "cluster_runId": "5f02e43b53a73a48d0eaaed5",
+        "cluster_runId": "5f033232366be85ec1afca7b",
         "cluster_connClustDict": {},
         "cluster_connNodesDict": [],
         "cluster_containedNodesDict": []
@@ -2894,7 +2894,7 @@
     {
         "cluster_label": 902,
         "cluster_layer": "FinishedTime_Layer",
-        "cluster_runId": "5f02e43b53a73a48d0eaaed5",
+        "cluster_runId": "5f033232366be85ec1afca7b",
         "cluster_connClustDict": {},
         "cluster_connNodesDict": [],
         "cluster_containedNodesDict": []
@@ -2902,7 +2902,7 @@
     {
         "cluster_label": 4501,
         "cluster_layer": "FinishedTime_Layer",
-        "cluster_runId": "5f02e43b53a73a48d0eaaed5",
+        "cluster_runId": "5f033232366be85ec1afca7b",
         "cluster_connClustDict": {},
         "cluster_connNodesDict": [],
         "cluster_containedNodesDict": []
@@ -2910,7 +2910,7 @@
     {
         "cluster_label": 2688,
         "cluster_layer": "FinishedTime_Layer",
-        "cluster_runId": "5f02e43b53a73a48d0eaaed5",
+        "cluster_runId": "5f033232366be85ec1afca7b",
         "cluster_connClustDict": {},
         "cluster_connNodesDict": [],
         "cluster_containedNodesDict": []
@@ -2918,7 +2918,7 @@
     {
         "cluster_label": 2873,
         "cluster_layer": "FinishedTime_Layer",
-        "cluster_runId": "5f02e43b53a73a48d0eaaed5",
+        "cluster_runId": "5f033232366be85ec1afca7b",
         "cluster_connClustDict": {},
         "cluster_connNodesDict": [],
         "cluster_containedNodesDict": []
@@ -2926,7 +2926,7 @@
     {
         "cluster_label": 3678,
         "cluster_layer": "FinishedTime_Layer",
-        "cluster_runId": "5f02e43b53a73a48d0eaaed5",
+        "cluster_runId": "5f033232366be85ec1afca7b",
         "cluster_connClustDict": {},
         "cluster_connNodesDict": [],
         "cluster_containedNodesDict": []
@@ -2934,7 +2934,7 @@
     {
         "cluster_label": 3683,
         "cluster_layer": "FinishedTime_Layer",
-        "cluster_runId": "5f02e43b53a73a48d0eaaed5",
+        "cluster_runId": "5f033232366be85ec1afca7b",
         "cluster_connClustDict": {},
         "cluster_connNodesDict": [],
         "cluster_containedNodesDict": []
@@ -2942,7 +2942,7 @@
     {
         "cluster_label": 677,
         "cluster_layer": "FinishedTime_Layer",
-        "cluster_runId": "5f02e43b53a73a48d0eaaed5",
+        "cluster_runId": "5f033232366be85ec1afca7b",
         "cluster_connClustDict": {},
         "cluster_connNodesDict": [],
         "cluster_containedNodesDict": []
@@ -2950,7 +2950,7 @@
     {
         "cluster_label": 843,
         "cluster_layer": "FinishedTime_Layer",
-        "cluster_runId": "5f02e43b53a73a48d0eaaed5",
+        "cluster_runId": "5f033232366be85ec1afca7b",
         "cluster_connClustDict": {},
         "cluster_connNodesDict": [],
         "cluster_containedNodesDict": []
@@ -2958,7 +2958,7 @@
     {
         "cluster_label": 4504,
         "cluster_layer": "FinishedTime_Layer",
-        "cluster_runId": "5f02e43b53a73a48d0eaaed5",
+        "cluster_runId": "5f033232366be85ec1afca7b",
         "cluster_connClustDict": {},
         "cluster_connNodesDict": [],
         "cluster_containedNodesDict": []
@@ -2966,7 +2966,7 @@
     {
         "cluster_label": 2119,
         "cluster_layer": "FinishedTime_Layer",
-        "cluster_runId": "5f02e43b53a73a48d0eaaed5",
+        "cluster_runId": "5f033232366be85ec1afca7b",
         "cluster_connClustDict": {},
         "cluster_connNodesDict": [],
         "cluster_containedNodesDict": []
@@ -2974,7 +2974,7 @@
     {
         "cluster_label": 0,
         "cluster_layer": "Destination_Layer",
-        "cluster_runId": "5f02e43b53a73a48d0eaaed5",
+        "cluster_runId": "5f033232366be85ec1afca7b",
         "cluster_connClustDict": {
             "-1": 57,
             "0": 21
@@ -4299,7 +4299,7 @@
     {
         "cluster_label": -1,
         "cluster_layer": "Destination_Layer",
-        "cluster_runId": "5f02e43b53a73a48d0eaaed5",
+        "cluster_runId": "5f033232366be85ec1afca7b",
         "cluster_connClustDict": {},
         "cluster_connNodesDict": [],
         "cluster_containedNodesDict": []
@@ -4307,7 +4307,7 @@
     {
         "cluster_label": 701,
         "cluster_layer": "Destination_Layer",
-        "cluster_runId": "5f02e43b53a73a48d0eaaed5",
+        "cluster_runId": "5f033232366be85ec1afca7b",
         "cluster_connClustDict": {},
         "cluster_connNodesDict": [],
         "cluster_containedNodesDict": []
@@ -4315,7 +4315,7 @@
     {
         "cluster_label": 319,
         "cluster_layer": "Destination_Layer",
-        "cluster_runId": "5f02e43b53a73a48d0eaaed5",
+        "cluster_runId": "5f033232366be85ec1afca7b",
         "cluster_connClustDict": {},
         "cluster_connNodesDict": [],
         "cluster_containedNodesDict": []
@@ -4323,7 +4323,7 @@
     {
         "cluster_label": 2230,
         "cluster_layer": "Destination_Layer",
-        "cluster_runId": "5f02e43b53a73a48d0eaaed5",
+        "cluster_runId": "5f033232366be85ec1afca7b",
         "cluster_connClustDict": {},
         "cluster_connNodesDict": [],
         "cluster_containedNodesDict": []
@@ -4331,7 +4331,7 @@
     {
         "cluster_label": 316,
         "cluster_layer": "Destination_Layer",
-        "cluster_runId": "5f02e43b53a73a48d0eaaed5",
+        "cluster_runId": "5f033232366be85ec1afca7b",
         "cluster_connClustDict": {},
         "cluster_connNodesDict": [],
         "cluster_containedNodesDict": []
@@ -4339,7 +4339,7 @@
     {
         "cluster_label": 2234,
         "cluster_layer": "Destination_Layer",
-        "cluster_runId": "5f02e43b53a73a48d0eaaed5",
+        "cluster_runId": "5f033232366be85ec1afca7b",
         "cluster_connClustDict": {},
         "cluster_connNodesDict": [],
         "cluster_containedNodesDict": []
@@ -4347,7 +4347,7 @@
     {
         "cluster_label": 1890,
         "cluster_layer": "Destination_Layer",
-        "cluster_runId": "5f02e43b53a73a48d0eaaed5",
+        "cluster_runId": "5f033232366be85ec1afca7b",
         "cluster_connClustDict": {},
         "cluster_connNodesDict": [],
         "cluster_containedNodesDict": []
@@ -4355,7 +4355,7 @@
     {
         "cluster_label": 1075,
         "cluster_layer": "Destination_Layer",
-        "cluster_runId": "5f02e43b53a73a48d0eaaed5",
+        "cluster_runId": "5f033232366be85ec1afca7b",
         "cluster_connClustDict": {},
         "cluster_connNodesDict": [],
         "cluster_containedNodesDict": []
@@ -4363,7 +4363,7 @@
     {
         "cluster_label": 1523,
         "cluster_layer": "Destination_Layer",
-        "cluster_runId": "5f02e43b53a73a48d0eaaed5",
+        "cluster_runId": "5f033232366be85ec1afca7b",
         "cluster_connClustDict": {},
         "cluster_connNodesDict": [],
         "cluster_containedNodesDict": []
@@ -4371,7 +4371,7 @@
     {
         "cluster_label": 2232,
         "cluster_layer": "Destination_Layer",
-        "cluster_runId": "5f02e43b53a73a48d0eaaed5",
+        "cluster_runId": "5f033232366be85ec1afca7b",
         "cluster_connClustDict": {},
         "cluster_connNodesDict": [],
         "cluster_containedNodesDict": []
@@ -4379,7 +4379,7 @@
     {
         "cluster_label": 2237,
         "cluster_layer": "Destination_Layer",
-        "cluster_runId": "5f02e43b53a73a48d0eaaed5",
+        "cluster_runId": "5f033232366be85ec1afca7b",
         "cluster_connClustDict": {},
         "cluster_connNodesDict": [],
         "cluster_containedNodesDict": []
@@ -4387,7 +4387,7 @@
     {
         "cluster_label": 1495,
         "cluster_layer": "Destination_Layer",
-        "cluster_runId": "5f02e43b53a73a48d0eaaed5",
+        "cluster_runId": "5f033232366be85ec1afca7b",
         "cluster_connClustDict": {},
         "cluster_connNodesDict": [],
         "cluster_containedNodesDict": []
@@ -4395,7 +4395,7 @@
     {
         "cluster_label": 526,
         "cluster_layer": "Destination_Layer",
-        "cluster_runId": "5f02e43b53a73a48d0eaaed5",
+        "cluster_runId": "5f033232366be85ec1afca7b",
         "cluster_connClustDict": {},
         "cluster_connNodesDict": [],
         "cluster_containedNodesDict": []
@@ -4403,7 +4403,7 @@
     {
         "cluster_label": 951,
         "cluster_layer": "Destination_Layer",
-        "cluster_runId": "5f02e43b53a73a48d0eaaed5",
+        "cluster_runId": "5f033232366be85ec1afca7b",
         "cluster_connClustDict": {},
         "cluster_connNodesDict": [],
         "cluster_containedNodesDict": []
@@ -4411,7 +4411,7 @@
     {
         "cluster_label": 2231,
         "cluster_layer": "Destination_Layer",
-        "cluster_runId": "5f02e43b53a73a48d0eaaed5",
+        "cluster_runId": "5f033232366be85ec1afca7b",
         "cluster_connClustDict": {},
         "cluster_connNodesDict": [],
         "cluster_containedNodesDict": []
@@ -4419,7 +4419,7 @@
     {
         "cluster_label": 648,
         "cluster_layer": "Destination_Layer",
-        "cluster_runId": "5f02e43b53a73a48d0eaaed5",
+        "cluster_runId": "5f033232366be85ec1afca7b",
         "cluster_connClustDict": {},
         "cluster_connNodesDict": [],
         "cluster_containedNodesDict": []
@@ -4427,7 +4427,7 @@
     {
         "cluster_label": 466,
         "cluster_layer": "Destination_Layer",
-        "cluster_runId": "5f02e43b53a73a48d0eaaed5",
+        "cluster_runId": "5f033232366be85ec1afca7b",
         "cluster_connClustDict": {},
         "cluster_connNodesDict": [],
         "cluster_containedNodesDict": []
@@ -4435,7 +4435,7 @@
     {
         "cluster_label": 2160,
         "cluster_layer": "Destination_Layer",
-        "cluster_runId": "5f02e43b53a73a48d0eaaed5",
+        "cluster_runId": "5f033232366be85ec1afca7b",
         "cluster_connClustDict": {},
         "cluster_connNodesDict": [],
         "cluster_containedNodesDict": []
@@ -4443,7 +4443,7 @@
     {
         "cluster_label": 1091,
         "cluster_layer": "Destination_Layer",
-        "cluster_runId": "5f02e43b53a73a48d0eaaed5",
+        "cluster_runId": "5f033232366be85ec1afca7b",
         "cluster_connClustDict": {},
         "cluster_connNodesDict": [],
         "cluster_containedNodesDict": []

src/data-hub/role-stage-discovery-microservice/app/resultTimeExecN2999C60.txt

-StartTime: 2020-07-06 10:43:32.240013
-FinishTime: 2020-07-06 10:43:39.110333
+StartTime: 2020-07-06 16:16:11.525479
+FinishTime: 2020-07-06 16:16:18.213974
 
-PopulateWithNewNodes: 2.399582
-CalculateWeights: 4.422768
-CalculateSimilarity: 0.04797
-TotalTime: 6.87032
-RunId: 5f02e43b53a73a48d0eaaed5
\ No newline at end of file
+PopulateWithNewNodes: 2.206513
+CalculateWeights: 4.435216
+CalculateSimilarity: 0.046766
+TotalTime: 6.688495
+RunId: 5f033232366be85ec1afca7b
\ No newline at end of file

src/data-hub/role-stage-discovery-microservice/app/routes/connClusters.py

 from flask import request, Response
 from db.repository import Repository
-from routes.connRun import add_connected_run
-from processing.similarityFiles.miscFunctions import *
 
 repo = Repository()
 
-
-def add_conn_clusters(inputDict,runId):
-    ''' Stores connected_clusters in the database.
-
-    :param Dict() inputDict: Contains the data to insert
-
-    :param string runId: Id of the Run 
-
-    ''' 
-    
-    outputJSON = convertLayerDictToJSON(inputDict,runId)
-    repo.add_connected_clusters(outputJSON)
-
-
 def get_conn_clusters():
     ''' Gets connected_clusters from the database.
 

src/data-hub/role-stage-discovery-microservice/app/routes/connRun.py

@@ -7,34 +7,17 @@ import datetime
 repo = Repository()
 
 
-def add_connected_run():
-    '''
-        Inserts Run with current Time into the DB
-        
-        :returns: Returns the _id of the connected_run entry in the DB
-    
-        :rtype: string
-    '''    
-
-    currentTime = datetime.datetime.now()
-    runDict = {"Datetime" : str(currentTime)}
-    inserted_result = repo.add_connected_run(runDict)
-    return str(inserted_result.inserted_id)
-
-
-def get_connected_run(): ########TODO#################
-    ''' ##TODO## Gets Run from the database.
+def get_connected_run():
+    ''' Gets Run from the database.
 
         :returns: Returns Run objects from the DB
         :rtype: Dict{_id,datetime}
     '''  
-    """
-    result = repo.get_connected_clusters()
-    if result is None or result.retrieved == 0:        
+    result = repo.get_connected_run()
+    if result is None or len(result) == 0:        
         print("#### Response 404")
         return Response(status=404)
     else:
         return result
-    conRun = ConnectedRun(result.sdfsdf)
-    """
+    
 

src/data-hub/role-stage-discovery-microservice/app/routes/similarity.py

@@ -5,19 +5,6 @@ from processing.similarityFiles.miscFunctions import convertSimilarityDictToJSON
 
 repo = Repository()
 
-
-def add_similarity(inputDict,runId):
-    ''' Stores cluster_similarity in the database.
-
-    :param Dict() inputDict: Contains the data to insert
-
-    :param string runId: Id of the Run 
-
-    ''' 
-
-    outputJSON = convertSimilarityDictToJSON(inputDict,runId)
-    repo.add_similarity(outputJSON)
-
 def get_similarity(): 
     ''' Gets cluster_similarity from the database.
 

src/data-hub/role-stage-discovery-microservice/app/similarityMain.py

@@ -88,7 +88,8 @@ def main():
 
     #Currently not used, developed for possible future uses
     connClustersFromMongo = getConnClusterDataFromMongo()    
-    similarityArrFromMongo = getSimilarityDataFromMOngo()
+    similarityArrFromMongo = getSimilarityDataFromMongo()
+    connectedRunFromMongo = getConnectedRunDataFromMongo()
     print("FINISHED")
 
     return