[RoleStage] Implemented selection for fetching and clustering scripts

src/data-hub/role-stage-discovery-microservice/app/processing/fetching/fetching.py

@@ -86,27 +86,31 @@ def _fetch_nodes(use_case: str, table: str, layer_name: str) -> List[Dict]:
     return response.json()
 
 
-def fetch_nodes_from_semantic_linking():
+def fetch_nodes_from_semantic_linking(selected_use_cases: List[str] = None, selected_use_case_tables: List[str] = None):
     '''Empties the db and inserts layers and nodes from BusinessLogic and SemanticLinking'''
     repository = Repository()
+    # please dont delete all layers/ nodes anymore @10.11.2020
     # repository.delete_all_layers()
     # repository.delete_all_nodes()
 
     use_cases = _fetch_use_cases()
     for use_case in use_cases:
+        if selected_use_cases is not None and use_case not in selected_use_cases:
+            continue
+
         print(f"Fetching for use-case {use_case}")
 
         tables = _fetch_tables(use_case)
         for table in tables:
-
-            if table != 'bank-app':
+            if selected_use_case_tables is not None and table not in selected_use_case_tables:
                 continue
-            
+
             layers = _fetch_layers(use_case, table)
             for layer in layers:
 
                 try:
                     print(f"Fetching nodes for layer {use_case}//{table}//{layer.layer_name}.")
+
                     # check if layer already exists in DB, add it if not
                     reference_layer = repository.get_layer_by_name(use_case, table, layer.layer_name)
                     if reference_layer == None:

src/data-hub/role-stage-discovery-microservice/app/convert_layers.py → src/data-hub/role-stage-discovery-microservice/app/read_json_layers.py

@@ -9,9 +9,11 @@ from db.entities import Cluster
 from typing import List
 from db.repository import Repository
 
+LAYER_FILES = ['User_Demand_Layer.json']
 
 repo = Repository()
 
+
 def get_clusters(layer_file) -> List[Cluster]:
     with open(layer_file, 'r') as file:
         clusters = json.loads(file.read())
@@ -19,10 +21,6 @@ def get_clusters(layer_file) -> List[Cluster]:
 
 
 def store_generic_clusters(clusters: List[Cluster], layer):
-
-    print([c.label for c in clusters][0:10])
-    return
-
     try:
         with open(f'{layer}.json', 'w') as file:
             cluster_dicts = [c.to_serializable_dict(for_db=False) for c in clusters]
@@ -37,9 +35,6 @@ def store_generic_clusters(clusters: List[Cluster], layer):
         print(f"failed uploading {layer}")
 
 
-layers = ['User_Demand_Layer.json']
-for layer in layers:
+for layer in LAYER_FILES:
     clusts: List[Cluster] = get_clusters(layer)
-    # print(len(clusts))
-
     store_generic_clusters(clusts, layer)
\ No newline at end of file

src/data-hub/role-stage-discovery-microservice/app/run_clustering.py

@@ -14,17 +14,20 @@ from processing.clustering import Clusterer, ClusterResult
 repo = Repository()
 
 
-def run_generic_clustering():
+def run_generic_clustering(selected_use_cases: List[str] = None, selected_use_case_tables: List[str] = None, selected_layer_names: List[str] = None):
     '''Runs the clustering for all layers found in the repository.'''
     all_layers:List[Layer] = repo.get_layers()
-    all_layers = [l for l in all_layers 
-        if l.layer_name in ['User_Demand_Layer'] and l.use_case == 'smart-energy']
+    layers = [l for l in all_layers 
+        if (selected_use_cases is None or l.use_case in selected_use_cases)
+        and (selected_use_case_tables is None or l.use_case_table in selected_use_case_tables)
+        and (selected_layer_names is None or l.layer_name in selected_layer_names)
+        ]
 
-    for layer in all_layers:        
+    for layer in layers:
         print(f"Clustering {layer.use_case}//{layer.use_case_table}//{layer.layer_name}.")
 
         if layer.properties is None or len(layer.properties) == 0:
-            print("skipping")
+            print("skipping, no properties to cluster")
             continue
 
         try:
@@ -54,15 +57,16 @@ def store_generic_clusters(clusters: List[Cluster], layer):
             cluster_dicts = [c.to_serializable_dict(for_db=False) for c in clusters]
             file.write(json.dumps(cluster_dicts))
     except:
-        pass
+        print(f"Error while writing json for {layer}")
 
     try:
         for cluster in clusters:
             repo.add_cluster(cluster)
     except:
-        pass
+        print(f"Error while storing cluster in db for {layer}")
 
 
 if __name__ == "__main__":
+    # please dont delete all clusters anymore @10.11.2020
     # repo.delete_all_clusters()
-    run_generic_clustering()
+    run_generic_clustering(selected_use_cases=[], selected_use_case_tables=[], selected_layer_names=[])

src/data-hub/role-stage-discovery-microservice/app/run_node_fetching.py

@@ -11,4 +11,4 @@ urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 import processing.fetching.fetching as f
 
 if __name__ == "__main__":
-    f.fetch_nodes_from_semantic_linking()
+    f.fetch_nodes_from_semantic_linking(selected_use_cases=[], selected_use_case_tables=[])
\ No newline at end of file

src/participation-hub/business-logic-microservice/app/_add_use_case_scripts/smart-energy/tables/add_smart_energy.py

@@ -166,7 +166,7 @@ def add_layers(use_case:str, table_name: str):
 
         print(url+": "+str(response.status_code))
 
-def main(use_case: str = "smart-energy", table_name: str = "smart-energy-paper"):
+def main(use_case: str = "smart-energy", table_name: str = "smart-energy"):
     print("SMART-ENERGY")
 
     add_table(use_case, table_name)