Modified a few files for visualisation

src/data-hub/role-stage-discovery-microservice/app/run_clustering.py

@@ -17,6 +17,24 @@ repo = Repository()
 def run_generic_clustering():
     '''Runs the clustering for all layers found in the repository.'''
     all_layers:List[Layer] = repo.get_layers()
+    #all_layers:List[Layer] = repo.get_layers_for_use_case("paper")
+
+    for layer in all_layers:
+        print(f"Clustering {layer.use_case}//{layer.use_case_table}//{layer.layer_name}.")
+
+        if layer.properties is None or len(layer.properties) == 0:
+            print("skipping")
+            continue
+
+        try:
+            clusters = run_clustering_for_layer(layer)
+            store_generic_clusters(clusters)
+        except Exception as e:
+            print(str(e))
+
+def run_clustering_for_Paper_case():
+
+    all_layers:List[Layer] = repo.get_layers_for_use_case("paper")
 
     for layer in all_layers:
         print(f"Clustering {layer.use_case}//{layer.use_case_table}//{layer.layer_name}.")

src/data-hub/role-stage-discovery-microservice/app/run_similarity_calc.py

@@ -6,6 +6,7 @@ repo = Repository()
 
 def run_similarity_calc_per_use_case():
     layers = repo.get_layers()
+    #layers = repo.get_layers_for_use_case("paper")
 
     uc_layers = {}
     for layer in layers:
@@ -19,6 +20,20 @@ def run_similarity_calc_per_use_case():
         print(f"Running for use case {key} with layers {str(layers)}.")
         SimilarityCalc.main(layerNameList=layers)
     
+def run_similarity_calc_for_Paper_case():
+    layers = repo.get_layers_for_use_case("paper")
+
+    uc_layers = {}
+    for layer in layers:
+        uc = layer.use_case
+        if uc not in uc_layers:
+            uc_layers[uc] = []
+        uc_layers[uc].append(layer.layer_name)
+
+    for key in uc_layers:
+        layers = uc_layers[key]
+        print(f"Running for use case {key} with layers {str(layers)}.")
+        SimilarityCalc.main(layerNameList=layers)
 
 if __name__ == '__main__':
     run_similarity_calc_per_use_case()
\ No newline at end of file

src/data-hub/role-stage-discovery-microservice/app/visualisationPaper.py

+# add modules folder to interpreter path
+# import sys
+# import os
+# modules_path = '../../../modules/'
+# if os.path.exists(modules_path):
+#     sys.path.insert(1, modules_path)
+# #######LOGGING############
+# import logging
+# LOG_FORMAT = ('%(levelname) -5s %(asctime)s %(name)s:%(funcName) -35s %(lineno) -5d:  %(message)s')
+# logging.basicConfig(level=logging.INFO, format=LOG_FORMAT)
+# LOGGER = logging.getLogger(__name__)
+
+######SECURITY#############
+# import connecion
+# from security import swagger_util
+# from pathlib import Path
+# from env_info import is_running_locally, get_resources_path
+
+######PROCESSING##########
+
+#from run_clustering import run_clustering_for_Paper_case
+import run_clustering as clustering
+import run_similarity_calc as similCalc
+
+import processing.similarityFiles.dataInput as inputData
+
+
+##################################
+#TODO: get certificates/authorization
+
+# SSL configuration
+# certificate_path = get_resources_path()
+# context = (os.path.normpath(f'{certificate_path}/articonf1.crt'), os.path.normpath(f'{certificate_path}/articonf1.key')) # certificate and key files
+
+# if is_running_locally():
+#     print("Running locally...")
+#     app.add_api(swagger_util.get_bundled_specs(Path("configs/swagger_local.yml")),
+#                 resolver = connexion.RestyResolver("cms_rest_api"))
+# else:
+#     app.add_api(swagger_util.get_bundled_specs(Path("configs/swagger.yml")),
+#                 resolver = connexion.RestyResolver("cms_rest_api"))
+
+##################################
+
+
+def mainViz():
+
+    #procesing data
+    clustering.run_clustering_for_Paper_case()
+    similCalc.run_similarity_calc_for_Paper_case()
+
+    inputData.getClusterDataFromMongo("Paper",None,None)
+    #inputData.getSimilarityDataFromMongo(cluster_layer: str= None, batchSize: int=1000, run_id: str=None)
+    #similarityArrFromMongo = getSimilarityDataFromMongo("Paper")

src/data-hub/semantic-linking-microservice/app/configs/routes.yml

@@ -38,6 +38,23 @@ paths:
           description: "Successful operation"
           schema:
             $ref: "#/definitions/NodeCollection"
+    delete: 
+      security:
+        - JwtAdmin: []
+      operationId: "routes.nodes.delete_nodes_for_use_case"
+      tags:
+        - "Nodes"
+      summary: "Delete all nodes in the Use-Case"
+      description: "Delete all Nodes in the specified Use-case"
+      parameters:
+        - name: "use_case"
+          in: "path"
+          description: "Name of the requested Use-Case"
+          required: true
+          type: "string"
+      responses:
+        '200':
+          description: "Successful Request"
   /use-cases/{use_case}/tables/{table_name}/nodes:
     get:
       security:
@@ -62,6 +79,30 @@ paths:
           description: "Successful operation"
           schema:
             $ref: "#/definitions/NodeCollection"
+    delete: 
+      security:
+        - JwtAdmin: []
+      operationId: "routes.nodes.delete_nodes_for_table"
+      tags:
+        - "Nodes"
+      summary: "Delete all nodes in the table of the Use-Case"
+      description: "Delete all Nodes in the table of the specified Use-case"
+      parameters:
+        - name: "use_case"
+          in: "path"
+          description: "Name of the requested Use-Case"
+          required: true
+          type: "string"
+        - name: "table_name"
+          in: "path"
+          description: "Name of Table the node belongs to"
+          required: true
+          type: "string"
+      responses:
+        '200':
+          description: "Successful Request"
+    
+      
   /use-cases/{use_case}/tables/{table_name}/layers/{layer_name}/nodes:
     get:
       security:
@@ -132,6 +173,28 @@ paths:
           description: "Successful operation"
           schema:
             $ref: "#/definitions/LayerCollection"
+    delete: 
+      security:
+        - JwtAdmin: []
+      operationId: "routes.layers.delete_layers_for_table"
+      tags:
+        - "Layers"
+      summary: "Delete all layers in the table of the Use-Case"
+      description: "Delete all layers in the table of the specified Use-case"
+      parameters:
+        - name: "use_case"
+          in: "path"
+          description: "Name of the requested Use-Case"
+          required: true
+          type: "string"
+        - name: "table_name"
+          in: "path"
+          description: "Name of Table"
+          required: true
+          type: "string"
+      responses:
+        '200':
+          description: "Successful Request"
       
 
   /use-cases/{use_case}/tables/{table_name}/layers/{name}:
@@ -187,6 +250,23 @@ paths:
             $ref: "#/definitions/LayerCollection"
         '404':
           description: "No layers found"
+    delete: 
+      security:
+        - JwtAdmin: []
+      operationId: "routes.layers.delete_layers_for_use_case"
+      tags:
+        - "Layers"
+      summary: "Delete all layers in the Use-Case"
+      description: "Delete all layers in the specified Use-case"
+      parameters:
+        - name: "use_case"
+          in: "path"
+          description: "Name of the requested Use-Case"
+          required: true
+          type: "string"
+      responses:
+        '200':
+          description: "Successful Request"
   # endregion layers
 
 definitions:

src/data-hub/semantic-linking-microservice/app/db/repository.py

@@ -66,6 +66,11 @@ class Repository(MongoRepositoryBase):
         collection = self._database[self._layer_collection]
         collection.delete_many({"use_case": use_case})
 
+    def delete_layers_for_table(self, use_case: str, table:str):
+        ''' Delete layers from the table of the given Use_case '''
+        collection = self._database[self._layer_collection]
+        collection.delete_many({"use_case": use_case, "table":table})
+
     def add_layer_node(self, node: dict):
         super().insert_entry(self._layer_nodes_collection, node)
 
@@ -95,4 +100,15 @@ class Repository(MongoRepositoryBase):
                                       projection={'_id': 0})
         return list(entries)
 
+    def delete_nodes_for_use_case(self, use_case):
+        '''Deletes All node of the given use_case'''
+        collection = self._database[self._layer_nodes_collection]
+        collection.delete_many({"use_case": use_case})
+
+    def delete_nodes_for_table(self, use_case, table_name):
+        '''Deletes nodes from the given table of the use-case'''
+        collection = self._database[self._layer_nodes_collection]
+        collection.delete_many({"use_case": use_case, "table": table_name})
+
+
     # endregion

src/data-hub/semantic-linking-microservice/app/routes/layers.py

@@ -33,4 +33,14 @@ def get_by_use_case(use_case: str):
     else:
         return Response(status=404)
 
+def delete_layers_for_use_case(use_case: str):
+    repo.delete_layers_for_use_case(use_case)
+    print("Deleted all layers from" + use_case)
+    return Response(status=200)
+
+def delete_layers_for_table(use_case: str, table_name: str):
+    repo.delete_layers_for_table(use_case, table_name)
+    print("Deleted all layers from: " + use_case + " , "+ table_name)
+    return Response(status=200)
+
 #endregion

src/data-hub/semantic-linking-microservice/app/routes/nodes.py

@@ -21,3 +21,13 @@ def nodes_for_use_case_and_table(use_case: str, table_name: str) -> List[Dict]:
 def nodes_for_use_case_and_table_and_layer(layer_name: str, use_case: str, table_name: str) -> List[Dict]:
     res = repo.get_nodes_for_use_case_and_table_and_layer(layer_name, use_case, table_name)
     return res
+
+def delete_nodes_for_use_case(use_case: str):
+    repo.delete_nodes_from_use_case(use_case)
+    print("Deleted all nodes from" + use_case)
+    return Response(status=200)
+
+def delete_nodes_for_table(use_case: str, table_name: str):
+    repo.delete_nodes_for_table(use_case, table_name)
+    print("Deleted all nodes from: " + use_case + " , "+ table_name)
+    return Response(status=200)
\ No newline at end of file