Changed pipeline to generate local files

src/data-hub/semantic-linking-microservice/app/dummy_upload/community-prediction-taxi/.gitignore

 # this file contains all the training data but is around 1.8 GB.
-train.csv
\ No newline at end of file
+train.csv
+
+# this folder contains all nodes generated from the upload script
+nodes/
\ No newline at end of file

src/data-hub/semantic-linking-microservice/app/dummy_upload/community-prediction-taxi/dummy_upload.py

@@ -93,4 +93,12 @@ if __name__ == '__main__':
         upload_transaction(transaction)
 
         if idx % 1000 == 0:
-            print(f"Progress: {str(float(idx) / 1710671)} %")
\ No newline at end of file
+            print(f"Progress: {str(float(idx) / 1710671)} %")
+            
+
+    all_nodes = handler.get_all()
+    layers = {n['layer_name'] for n in all_nodes}
+    for l in layers:
+        layer_content = [e for e in all_nodes if e['layer_name'] == l]
+        with open(f'{l}.json', 'w') as f:
+            f.write(json.dumps(layer_content))
\ No newline at end of file

src/data-hub/semantic-linking-microservice/app/dummy_upload/community-prediction-youtube/.gitignore

 # this file contains all the training data but is large.
-videos.csv
\ No newline at end of file
+videos.csv
+
+# this folder contains all nodes generated from the upload script
+nodes/
\ No newline at end of file

src/data-hub/semantic-linking-microservice/app/dummy_upload/community-prediction-youtube/dummy_upload.py

@@ -41,8 +41,8 @@ def upload_transaction(transaction):
     del transaction['trending_timestamp']
 
     t = {        
-        'use_case':  'community-prediction-youtube-n',
-        'table':  'community-prediction-youtube-n',
+        'use_case':  'community-prediction-youtube',
+        'table':  'community-prediction-youtube',
         'id': uid,
         'properties': transaction,
     }
@@ -57,4 +57,8 @@ if __name__ == '__main__':
         upload_transaction(transaction)
 
         if idx % 1000 == 0:
-            print(f"Progress: {str(float(idx) / 375942)} %")
\ No newline at end of file
+            print(f"Progress: {str(float(idx) / 375942)} %")
+
+    res = handler.get_all()
+    with open("res.json", 'w') as f:
+        f.write((json.dumps(res)))
\ No newline at end of file

src/data-hub/semantic-linking-microservice/app/messaging/MessageHandler.py

@@ -13,6 +13,7 @@ LOGGER = logging.getLogger(__name__)
 class MessageHandler:
     def __init__(self, repository):
         self._repository = repository
+        self.allnodes = []
 
     def handle_generic(self, body):
         LOGGER.info(f"Received message: {body}")
@@ -97,7 +98,14 @@ class MessageHandler:
             nodes.append(node)
         
         if len(nodes) > 0:
-            self._repository.add_layer_nodes(nodes)
+            self.store(nodes)
+            # self._repository.add_layer_nodes(nodes)
+        
+    def store(self, nodes):
+        self.allnodes.extend(nodes)
+
+    def get_all(self):
+        return self.allnodes
 
     def handle_new_traces_available(self):
         # get all traces and call the Processor

src/data-hub/semantic-linking-microservice/app/requirements.txt

@@ -22,6 +22,7 @@ lazy-object-proxy==1.4.3
 MarkupSafe==1.1.1
 mccabe==0.6.1
 openapi-spec-validator==0.2.8
+pandas
 pika==1.1.0
 prance==0.19.0
 pycparser==2.20

src/participation-hub/business-logic-microservice/app/_add_use_case_scripts/community-prediction/taxi/add_schema.py → src/participation-hub/business-logic-microservice/app/_add_use_case_scripts/community_prediction/taxi/add_schema.py

@@ -7,14 +7,17 @@ import requests
 modules_path = '../../../modules/'
 if os.path.exists(modules_path):
     sys.path.insert(1, modules_path)
+sys.path.insert(1, '.')
 
 import network_constants as nc
 from security.token_manager import TokenManager
 
-import tables.add_table as add_table
+import _add_use_case_scripts.community_prediction.taxi.tables.add_table as add_table
+# import tables.add_table as add_table
+
 
 def add_use_case(use_case: str):
-    jwt = TokenManager.getInstance().getToken()
+    jwt = "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJ1c2VybmFtZSI6InJvb3RAaXRlYy5hYXUuYXQiLCJjcmVhdGVkX2F0IjoiMjAyMS0xMi0wOCAxNzo1OToxOC43OTMxMzUiLCJ2YWxpZF91bnRpbCI6IjIwMjEtMTItMDkgMTc6NTk6MTguNzkzMTM1In0.yQPH0KL3oaMyJAj_QcdEaSEWgSHSI-2qpck_aa3czHk"
     url = f"https://articonf1.itec.aau.at:30420/api/use-cases"
     response = requests.post(
             url,
@@ -24,7 +27,7 @@ def add_use_case(use_case: str):
             json = {"name": use_case}
         )
     
-    print(url+": "+str(response.status_code))
+    print(url+": "+str(response.content))
         
 
 if __name__ == "__main__":

src/participation-hub/business-logic-microservice/app/_add_use_case_scripts/community-prediction/taxi/tables/add_table.py → src/participation-hub/business-logic-microservice/app/_add_use_case_scripts/community_prediction/taxi/tables/add_table.py

@@ -6,7 +6,7 @@ import requests
 
 def add_table(use_case: str, table_name: str):
     ''' Adds the use-case table with all the mappings as dict Internal -> External. '''
-    jwt = TokenManager.getInstance().getToken()
+    jwt = "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJ1c2VybmFtZSI6InJvb3RAaXRlYy5hYXUuYXQiLCJjcmVhdGVkX2F0IjoiMjAyMS0xMi0wOCAxNzo1OToxOC43OTMxMzUiLCJ2YWxpZF91bnRpbCI6IjIwMjEtMTItMDkgMTc6NTk6MTguNzkzMTM1In0.yQPH0KL3oaMyJAj_QcdEaSEWgSHSI-2qpck_aa3czHk"
     
     mapping = { c : c for c in [
         # mapping does not change any of the names for these properties
@@ -44,7 +44,7 @@ def add_table(use_case: str, table_name: str):
 
 
 def add_layers(use_case:str, table_name: str):
-    jwt = TokenManager.getInstance().getToken()
+    jwt = "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJ1c2VybmFtZSI6InJvb3RAaXRlYy5hYXUuYXQiLCJjcmVhdGVkX2F0IjoiMjAyMS0xMi0wOCAxNzo1OToxOC43OTMxMzUiLCJ2YWxpZF91bnRpbCI6IjIwMjEtMTItMDkgMTc6NTk6MTguNzkzMTM1In0.yQPH0KL3oaMyJAj_QcdEaSEWgSHSI-2qpck_aa3czHk"
 
     layers = [
         {