Fix clustering/fetching.. added MongoRep funct

src/data-hub/role-stage-discovery-microservice/app/db/repository.py

@@ -57,8 +57,14 @@ class Repository(MongoRepositoryBase):
         else:
             return None
     
-    def delete_all_layers(self):
+    def delete_all_layers(self, use_case:str ):
         super().drop_collection(self._layer_collection)
+
+    def delete_use_case_layers(self, use_case: str):
+        collection_name = self._layer_collection
+        criteria = {"use_case" : use_case}
+        res = super().delete_entry(collection_name,criteria,True)
+        print ("###REPOSITORY: res= "+ str(res))
     
 #endregion Layers
 
@@ -78,6 +84,12 @@ class Repository(MongoRepositoryBase):
         entries = super().get_entries(self._layer_nodes_collection, selection={'use_case': use_case, 'use_case_table': use_case_table, 'layer_name': layer_name}, projection={'_id': 0})
         return [e for e in entries]
 
+    def delete_use_case_nodes(self, use_case: str):
+        collection_name = self._layer_nodes_collection
+        criteria = {"use_case" : use_case}
+        res = super().delete_entry(collection_name,criteria,True)
+        print ("###REPOSITORY: res= "+ str(res))
+
 #endregion
 
 #region Clusters

src/data-hub/role-stage-discovery-microservice/app/processing/fetching/fetching.py

@@ -89,11 +89,14 @@ def _fetch_nodes(use_case: str, table: str, layer_name: str) -> List[Dict]:
 def fetch_nodes_from_semantic_linking():
     '''Empties the db and inserts layers and nodes from BusinessLogic and SemanticLinking'''
     repository = Repository()
-    repository.delete_all_layers()
-    repository.delete_all_nodes()
+    # repository.delete_all_layers()
+    # repository.delete_all_nodes()
 
+    repository.delete_use_case_layers("paper")
+    repository.delete_use_case_nodes("paper")
+    
     use_cases = _fetch_use_cases()
-    for use_case in use_cases:
+    for use_case in [uc for uc in use_cases if uc == 'paper']:
         print(f"Fetching for use-case {use_case}")
 
         tables = _fetch_tables(use_case)

src/data-hub/role-stage-discovery-microservice/app/visualisationPaper.py

-# add modules folder to interpreter path
-# import sys
-# import os
-# modules_path = '../../../modules/'
-# if os.path.exists(modules_path):
-#     sys.path.insert(1, modules_path)
-# #######LOGGING############
-# import logging
-# LOG_FORMAT = ('%(levelname) -5s %(asctime)s %(name)s:%(funcName) -35s %(lineno) -5d:  %(message)s')
-# logging.basicConfig(level=logging.INFO, format=LOG_FORMAT)
-# LOGGER = logging.getLogger(__name__)
+###add modules folder to interpreter path
+import sys
+import os
+modules_path = '../../../modules/'
+if os.path.exists(modules_path):
+    sys.path.insert(1, modules_path)
+
 
 ######SECURITY#############
 # import connecion
@@ -18,9 +14,9 @@
 
 ######PROCESSING##########
 
-#from run_clustering import run_clustering_for_Paper_case
 import run_clustering as clustering
 import run_similarity_calc as similCalc
+import processing.fetching.fetching as fetchy
 
 import processing.similarityFiles.dataInput as inputData
 
@@ -47,9 +43,13 @@ import processing.similarityFiles.dataInput as inputData
 def mainViz():
 
     #procesing data
-    clustering.run_clustering_for_Paper_case()
+    #fetchy.fetch_nodes_from_semantic_linking()
+    #clustering.run_clustering_for_Paper_case()
     similCalc.run_similarity_calc_for_Paper_case()
 
     inputData.getClusterDataFromMongo("Paper",None,None)
     #inputData.getSimilarityDataFromMongo(cluster_layer: str= None, batchSize: int=1000, run_id: str=None)
     #similarityArrFromMongo = getSimilarityDataFromMongo("Paper")
+
+
+mainViz()
\ No newline at end of file

src/modules/database/MongoRepositoryBase.py

@@ -15,6 +15,9 @@ class MongoRepositoryBase:
     def drop_collection(self, collection_name):
         self._database[collection_name].drop()
 
+    def delete_entry(self, collection_name, criteria: dict, multi: bool):
+        self._database[collection_name].remove(criteria, multi)
+
     def insert_entry(self, collection_name, content: dict):
         collection = self._database[collection_name]
         return collection.insert_one(content)