[RoleStage] Split upload of similarity results due to size

src/data-hub/role-stage-discovery-microservice/app/db/repository.py

@@ -104,6 +104,9 @@ class Repository(MongoRepositoryBase):
 #endregion
 
 #region clusterConnected
+    def add_connected_cluster(self, clusterDict):
+        super().insert_entry(self._connected_clusters_collection, clusterDict)
+
     def add_connected_clusters(self, clusterDictArray):
         ''' Add Connected Clusters Data to DB '''
         result = super().insert_many(self._connected_clusters_collection, clusterDictArray)
@@ -126,6 +129,9 @@ class Repository(MongoRepositoryBase):
 #endregion
 
 #region similarity
+    def add_single_similarity(self, input_):
+        super().insert_entry(self._similarity_collection, input_)
+
     def add_similarity(self, inputDict):
         ''' Add Similarity Data to DB '''
         #checkIfConnClustDictIsSerializable(outputJSON)      

src/data-hub/role-stage-discovery-microservice/app/processing/similarityFiles/dataOutput.py

@@ -126,7 +126,8 @@ def add_conn_clusters(inputDict,runId):
     ''' 
     
     outputJSON = convertLayerDictToJSON(inputDict,runId)
-    repo.add_connected_clusters(outputJSON)
+    for element in outputJSON:
+        repo.add_connected_cluster(element)
 
 def add_similarity(inputDict,runId):
     ''' Stores cluster_similarity in the database.
@@ -138,4 +139,5 @@ def add_similarity(inputDict,runId):
     ''' 
 
     outputJSON = convertSimilarityDictToJSON(inputDict,runId)
-    repo.add_similarity(outputJSON)
\ No newline at end of file
+    for element in outputJSON:
+        repo.add_single_similarity(element)
\ No newline at end of file

src/data-hub/role-stage-discovery-microservice/app/processing/similarityFiles/similarityMain.py

@@ -37,6 +37,7 @@ from processing.similarityFiles.calculateSimilarity import *
 from processing.similarityFiles.miscFunctions import *
 from processing.similarityFiles.dataOutput import *
 
+outputToFileFLAG = True
 
 def main(layerNameList:List[str] = ["Price_Layer","FinishedTime_Layer","Destination_Layer"]):   
     '''
@@ -48,8 +49,6 @@ def main(layerNameList:List[str] = ["Price_Layer","FinishedTime_Layer","Destinat
     '''
     print("Entered Similarity Main")
 
-    outputToFileFLAG = False
-
     timelist = []
     timelist.append(currentTime())#starting time
     

src/transaction-hub-in/trace-retrieval-microservice/app/database/repository.py

@@ -12,9 +12,9 @@ class Repository(MongoRepositoryBase):
     '''This is a repository for MongoDb.'''
 
     def __init__(self):
-        super().__init__(netconst.BUSINESS_LOGIC_DB_HOSTNAME,
-                         netconst.BUSINESS_LOGIC_DB_PORT,
-                         'rest-gateway-db')
+        super().__init__(netconst.TRACE_RETRIEVAL_DB_HOSTNAME,
+                         netconst.TRACE_RETRIEVAL_DB_PORT,
+                         'traceRetrievalDb')
 
         self._transaction_collection = 'transactions'
         self._failed_transaction_collection = 'transactions_failed'