[RoleStage] Error handling when fetching from semantic-linking

src/data-hub/role-stage-discovery-microservice/app/processing/fetching/fetching.py

@@ -20,7 +20,7 @@ def _fetch_layers(use_case: str) -> List[Layer]:
     )
 
     if response.status_code != 200:
-        raise ConnectionError(f"Could not fetch data from the server, statuscode: {response.status_code}!")
+        raise ConnectionError(f"Could not fetch layers for {use_case} from business-logic microservice, statuscode: {response.status_code}!")
 
     data = json.loads(response.text)
 
@@ -44,7 +44,7 @@ def _fetch_nodes(use_case: str, layer: Layer) -> List[Dict]:
     )
 
     if response.status_code != 200:
-        raise ConnectionError(f"Could not fetch data from the server, statuscode: {response.status_code}!")
+        raise ConnectionError(f"Could not fetch nodes for {use_case}/{layer.layer_name} from semantic-linking microservice, statuscode: {response.status_code}!")
 
     data = json.loads(response.text)
 
@@ -56,7 +56,6 @@ def _fetch_nodes(use_case: str, layer: Layer) -> List[Dict]:
     return nodes
 
 def _fetch_use_cases() -> List[str]:
-    # fetch use-cases
     jwt = TokenManager.getInstance().getToken()
 
     url = f'https://{network_constants.BUSINESS_LOGIC_HOSTNAME}:{network_constants.BUSINESS_LOGIC_REST_PORT}/api/use-cases'
@@ -69,31 +68,36 @@ def _fetch_use_cases() -> List[str]:
     )
 
     if response.status_code != 200:
-        raise ConnectionError(f"Could not fetch data from the server, statuscode: {response.status_code}!")
+        raise ConnectionError(f"Could not fetch use-cases from business-logic microservice, statuscode: {response.status_code}!")
 
     data = json.loads(response.text)
 
     return [row["name"] for row in data]
 
 def fetch_nodes_from_semantic_linking():
-    use_cases = _fetch_use_cases()
+    # prune DB 
+    repository = Repository()
+    repository.delete_all_layers()
+    repository.delete_all_nodes()
 
+    use_cases = _fetch_use_cases()
     for use_case in use_cases:
-        # prune DB 
-        repository = Repository()
-        repository.delete_all_layers()
-        repository.delete_all_nodes()
-
+        print(f"Fetching for use-case {use_case}")
         layers = _fetch_layers(use_case)
 
         for layer in layers:
-            # check if layer already exists in DB, add it if not
-            reference_layer = repository.get_layer_for_use_case(layer.layer_name, use_case)
-            if reference_layer == None:
-                repository.add_layer(layer)
-
-            nodes = repository.get_layer_nodes(layer.layer_name)
-            nodes = _fetch_nodes(use_case, layer)
-
-            for node in nodes:
-                repository.add_layer_node(node)
\ No newline at end of file
+            try:
+                print(f"Fetching nodes for layer {layer.layer_name}")
+                # check if layer already exists in DB, add it if not
+                reference_layer = repository.get_layer_for_use_case(layer.layer_name, use_case)
+                if reference_layer == None:
+                    repository.add_layer(layer)
+
+                nodes = _fetch_nodes(use_case, layer)
+
+                for node in nodes:
+                    repository.add_layer_node(node)
+            except ConnectionError as e:
+                print(str(e))
+                continue
+            
\ No newline at end of file

src/data-hub/role-stage-discovery-microservice/app/run_node_fetching.py

@@ -9,5 +9,4 @@ import json
 import processing.fetching.fetching as f
 
 if __name__ == "__main__":
-    # TODO read use_case from somewhere
     f.fetch_nodes_from_semantic_linking()
\ No newline at end of file