Merge branch 'feature/enums' into develop

src/data-hub/semantic-linking-microservice/app/messaging/MessageHandler.py

@@ -54,15 +54,21 @@ class MessageHandler:
         
         layers = [Layer.from_business_logic_dict(row) for row in json.loads(response.text)]
 
+        LOGGER.info(f"Received {len(layers)} layers")
+
         # update local DB, insert each layer that does not already exists
         for layer in layers:
             print(f"Add layer to DB: {layer.to_serializable_dict(for_db=True)}")
             self._repository.delete_layer(layer)
             self._repository.add_layer(layer)
+        
+        if len(layers)==0:
+            LOGGER.error(f"no schema information found")
 
         return layers
 
     def handle_new_trace(self, content: Dict):
+        LOGGER.info("new trace!")
         if "use_case" not in content or "id" not in content or "properties" not in content or "table" not in content:
             LOGGER.error(f"Missing fields in trace, required fields: (use_case, id, properties, table), given fields: ({content.keys()})")
             return
@@ -82,6 +88,7 @@ class MessageHandler:
             LOGGER.warning(f"No layers available for '{use_case}'.'{table}', ignoring trace.")
             return
 
+        LOGGER.info(f"{len(layers)} layers available")
         nodes = []
         
         for layer in layers:
@@ -97,7 +104,12 @@ class MessageHandler:
             nodes.append(node)
         
         if len(nodes) > 0:
+            LOGGER.info(f"{len(layers)} layers available")
             self._repository.add_layer_nodes(nodes)
+        else:
+            LOGGER.error(f"did NOT add nodes...")
+        
+        LOGGER.info("done")
 
     def handle_new_traces_available(self):
         # get all traces and call the Processor

src/participation-hub/business-logic-microservice/app/_add_use_case_scripts/crowd_journalism/add_crowdjournalism_schema.py

@@ -29,7 +29,7 @@ def add_use_case(use_case: str):
     print(url+": "+str(response.content))     
 
 if __name__ == "__main__":
-    use_case = "crowd-journalism"
+    use_case = "crowd-journalism-enum"
     
     # disable ssl warnings :) 
     requests.packages.urllib3.disable_warnings() 

src/participation-hub/business-logic-microservice/app/_add_use_case_scripts/crowd_journalism/tables/add_classification.py

@@ -18,6 +18,7 @@ def add_table(use_case: str, table_name: str):
     
     columns = { c : c for c in columns }
     columns["UniqueID"] = "userid+videoid"
+    columns["objecttype"] = "enum(objecttype)"
 
     table = {
         "name": table_name,

src/participation-hub/business-logic-microservice/app/_add_use_case_scripts/crowd_journalism/tables/add_event.py

@@ -17,7 +17,8 @@ def add_table(use_case: str, table_name: str):
     
     columns = { c : c for c in columns }
     columns["UniqueID"] = "eventid"
-    columns["firstTag"] = "tags[0]"
+    columns["firstTag"] = "enum(tags[0])"
+    columns["objecttype"] = "enum(objecttype)"
 
     table = {
         "name": table_name,

src/participation-hub/business-logic-microservice/app/_add_use_case_scripts/crowd_journalism/tables/add_purchase.py

@@ -17,6 +17,10 @@ def add_table(use_case: str, table_name: str):
     
     columns = { c : c for c in columns }
     columns["UniqueID"] = "userid+videoid+ownerid"
+    columns["objecttype"] = "enum(objecttype)"
+    columns["userid"] = "enum(userid)"
+    columns["ownerid"] = "enum(ownerid)"
+
 
     table = {
         "name": table_name,

src/participation-hub/business-logic-microservice/app/_add_use_case_scripts/crowd_journalism/tables/add_tag.py

@@ -5,13 +5,9 @@ def add_table(use_case: str, table_name: str):
     replace all "/"'s in the internal representation with a "_"
     '''
 
-    columns = [
-        # "docType",
-        "objecttype",
-        "tag"
-    ]
-    
-    columns = { c : c for c in columns }
+    columns = {}
+    columns["tag"] = "enum(tag)"
+    columns["objecttype"] = "enum(objecttype)"
     columns["UniqueID"] = "objecttype+tag"
 
     table = {

src/participation-hub/business-logic-microservice/app/_add_use_case_scripts/crowd_journalism/tables/add_video.py

@@ -33,7 +33,10 @@ def add_table(use_case: str, table_name: str):
     columns["UniqueID"] = "videoid"
     columns["encodedAudio"] = "codec//audio"
     columns["encodedVideo"] = "codec//video"
-    columns["firstTag"] = "tags[0]"
+
+    columns["objecttype"] = "enum(objecttype)"
+    columns["duration"] = "enum(duration)"
+    columns["firstTag"] = "enum(tags[0])"
 
     table = {
         "name": table_name,

src/participation-hub/business-logic-microservice/app/_add_use_case_scripts/debug/sample_trace.json

+{
+    "ApplicationType": "debug",
+    "docType": "pizza",
+    "id": 1,
+    "dough": {
+        "type": "wheat",
+        "spinach": false
+    },
+    "toppings": [
+        {
+            "name": "cheese",
+            "price": 0.99
+        }
+    ],
+    "name": "Margherita"
+}
\ No newline at end of file

src/participation-hub/business-logic-microservice/app/_add_use_case_scripts/debug/tables/add_pizza_table.py

@@ -64,6 +64,7 @@ def main(use_case: str):
         {
             "UniqueID": "id",
             "name": "name",
+            "nameIndex": "enum(name)",
             "doughType": "dough//type",
             "hasSpinach": "dough//spinach",
             "firstTopping": "toppings[0]//name",
@@ -81,10 +82,11 @@ def main(use_case: str):
                 "name": "Price_Layer",
                 "properties": [
                     "UniqueID",
-                    "firstToppingPrice"
+                    "firstToppingPrice",
+                    "nameIndex"
                 ],
                 "cluster_properties": [
-                    "firstToppingPrice"
+                    "firstToppingPrice",
                 ]
             },
             {

src/participation-hub/business-logic-microservice/app/clone_use_case.py

+import sys
+import os
+import json
+from pathlib import Path
+from typing import Dict, Any
+
+import requests
+requests.packages.urllib3.disable_warnings() 
+
+modules_paths = ['.', '../../../modules/']
+for modules_path in modules_paths:
+    if os.path.exists(modules_path):
+        sys.path.insert(1, modules_path)
+
+import network_constants as nc
+from security.token_manager import TokenManager
+
+use_case_from = "crowd-journalism"
+use_case_to = "crowd-journalism-enum"
+
+
+# 1. get all tables for the use-case
+from db.entities.table import Table
+
+jwt = TokenManager.getInstance().getToken()
+url = f"https://articonf1.itec.aau.at:30420/api/use-cases/{use_case_from}/tables"
+response = requests.get(
+        url,
+        verify=False, 
+        proxies = { "http":None, "https":None },
+        headers = { "Authorization": f"Bearer {jwt}"},
+    )
+
+tables = json.loads(response.content)
+tables = [Table.from_serializable_dict(row) for row in tables]
+
+print(f"|tables|={len(tables)}")
+
+
+#2. fetch layers for tables
+from db.entities.layer_adapter import LayerAdapter
+
+url = f"https://articonf1.itec.aau.at:30420/api/use-cases/{use_case_from}/layers"
+response = requests.get(
+        url,
+        verify=False, 
+        proxies = { "http":None, "https":None },
+        headers = { "Authorization": f"Bearer {jwt}"},
+    )
+
+layers = json.loads(response.content)
+layers = [LayerAdapter.from_serializable_dict(row) for row in layers]
+
+print(f"|layers|={len(layers)}")
+
+
+# 3. fetch transactions for use-case
+
+url = f"https://articonf1.itec.aau.at:30001/api/use_cases/{use_case_from}/transactions"
+response = requests.get(
+        url,
+        verify=False, 
+        proxies = { "http":None, "https":None },
+        headers = { "Authorization": f"Bearer {jwt}"},
+    )
+
+transactions = json.loads(response.content)
+print(f"|transactions|={len(transactions)}")
+
+ignored_fields = ["ApplicationType","docType","UniqueID"]
+
+for table in tables:
+    for transaction in transactions:
+        if transaction["table"] != table.name:
+            continue
+
+        data = {
+            "ApplicationType": use_case_to,
+            "docType": table.name,
+        }
+
+        if table.name == "video":
+            data["codec"] = {
+                "audio": transaction["properties"]["encodedAudio"],
+                "video": transaction["properties"]["encodedVideo"]
+            }
+            data["tags"] = [transaction["properties"]["firstTag"]]
+        if table.name == "event":
+            data["tags"] = [transaction["properties"]["firstTag"]]
+
+
+        for key,value in transaction["properties"].items():
+            if key not in ignored_fields:
+                data[key] = value
+        
+        # post the new trace
+        url = f"https://articonf1.itec.aau.at:30401/api/trace"
+        response = requests.post(
+            url,
+            verify=False, 
+            proxies = { "http":None, "https":None },
+            headers = { "Authorization": f"Bearer {jwt}"},
+            json = data
+        )
+
+        print(url+": "+str(response.status_code))

src/participation-hub/business-logic-microservice/app/configs/routes.yml

@@ -515,6 +515,109 @@ paths:
           description: "Field in request is missing"
         '403':
           description: "Confirmation required"
+  #####
+  # END LAYERS
+  #####
+
+  #####
+  # ENUMS
+  #####
+  /enums:
+    get:
+      security:
+        - JwtRegular: []
+      operationId: "routes.enum.all"
+      tags:
+        - "Enums"
+      summary: "Retrieve all Enums from the DB"
+      description: "Retrieve all Enums from the DB"
+      responses:
+        '200':
+          description: "Successful Request"
+  /use-cases/{use_case}/enums:
+    get:
+      security:
+        - JwtRegular: []
+      operationId: "routes.enum.get_all_for_use_case"
+      tags:
+        - "Enums"
+      summary: "Retrieve one Enum from the DB"
+      description: "Retrieve one Enum from the DB"
+      parameters:
+        - name: "use_case"
+          in: "path"
+          description: "Name of the Use-Case the Enum belongs to"
+          required: true
+          type: "string"
+      responses:
+        '200':
+          description: "Successful Request"
+        '404':
+          description: "Enum does not exist"
+  /use-cases/{use_case}/tables/{table}/enum/{name}:
+    put:
+      security:
+        - JwtRegular: []
+      operationId: "routes.enum.put_new"
+      tags:
+        - "Enums"
+      summary: "Updates an existing Enum with a new value."
+      description: "Updates an existing Enum with a new value."
+      parameters:
+        - name: "use_case"
+          in: "path"
+          description: "Name of the Use-Case the Enum belongs to"
+          required: true
+          type: "string"
+        - name: "table"
+          in: "path"
+          description: "Name of the Table the Enum belongs to"
+          required: true
+          type: "string"
+        - name: "name"
+          in: "path"
+          description: "Name of the Enum"
+          required: true
+          type: "string"
+        - name: value
+          in: query
+          description: Value of the Enum. 
+          required: true
+          type: string
+      responses:
+        '200':
+          description: "Successful Request"
+        '404':
+          description: "Enum does not exist"
+    get:
+      security:
+        - JwtRegular: []
+      operationId: "routes.enum.one"
+      tags:
+        - "Enums"
+      summary: "Retrieve one Enum from the DB"
+      description: "Retrieve one Enum from the DB"
+      parameters:
+        - name: "use_case"
+          in: "path"
+          description: "Name of the Use-Case the Enum belongs to"
+          required: true
+          type: "string"
+        - name: "table"
+          in: "path"
+          description: "Name of the Table the Enum belongs to"
+          required: true
+          type: "string"
+        - name: "name"
+          in: "path"
+          description: "Name of the Enum"
+          required: true
+          type: "string"
+      responses:
+        '200':
+          description: "Successful Request"
+        '404':
+          description: "Enum does not exist"
 
 definitions:
   LayerMapping:

src/participation-hub/business-logic-microservice/app/db/entities/enum.py

+from typing import Dict
+
+
+class Enum:
+    def __init__(self, use_case: str, table: str, name: str, value: str, index: int):
+        self.use_case = use_case
+        self.table = table
+        self.name = name
+        self.value = value
+        self.index = index
+
+    def to_serializable_dict(self) -> Dict:
+        return {
+            "use_case": self.use_case,
+            "table": self.table,
+            "name": self.name,
+            "value": self.value,
+            "index": self.index,
+        }
+
+    @staticmethod
+    def from_serializable_dict(enum_dict: Dict):
+        '''
+        creates a layer object from a dictionary. has to have the following keys:
+            - name
+            - properties
+            - cluster_properties
+            - use_case
+        '''
+        return Enum(
+            enum_dict["use_case"],
+            enum_dict["table"],
+            enum_dict["name"],
+            enum_dict["value"],
+            enum_dict["index"],
+        )

src/participation-hub/business-logic-microservice/app/db/enum_repository.py

+# global imports (dont't worry, red is normal)
+import network_constants as netconst
+from database.MongoRepositoryBase import MongoRepositoryBase
+from db.entities.enum import Enum
+
+from typing import List
+
+
+class EnumRepository(MongoRepositoryBase):
+    '''This is a repository for MongoDb.'''
+
+    def __init__(self):
+        super().__init__(netconst.BUSINESS_LOGIC_DB_HOSTNAME,
+                         netconst.BUSINESS_LOGIC_DB_PORT,
+                         'business-logic-db')
+
+        self._enum_collection = 'enums'
+
+    def all(self) -> List[Enum]:
+        result = super().get_entries(
+            self._enum_collection, projection={'_id': False})
+
+        return [Enum.from_serializable_dict(row) for row in list(result)]
+
+    def all_for_use_case(self, use_case: str) -> List[Enum]:
+        result = super().get_entries(self._enum_collection, projection={
+            '_id': False}, selection={"use_case": use_case})
+
+        return [Enum.from_serializable_dict(row) for row in list(result)]
+
+    def all_for_use_case_and_table(self, use_case: str, table: str) -> List[Enum]:
+        result = super().get_entries(self._enum_collection, projection={
+            '_id': False}, selection={"use_case": use_case, "table": table})
+
+        return [Enum.from_serializable_dict(row) for row in list(result)]
+
+    def all_for_use_case_and_table_and_name(self, name: str, use_case: str, table: str) -> Enum:
+        result = list(super().get_entries(self._enum_collection, selection={
+                      "name": name, "use_case": use_case, "table": table}))
+
+        return [Enum.from_serializable_dict(row) for row in result]
+
+    def delete_all(self):
+        collection = self._database[self._enum_collection]
+        collection.delete_many({})
+
+    def add(self, enum: Enum):
+        super().insert_entry(self._enum_collection, enum.to_serializable_dict())
+
+    def update_use_case(self, enum: Enum, use_case: str):
+        collection = self._database[self._enum_collection]
+        collection.update_one({"name": enum.name, "use_case": use_case, "table": enum.table}, {
+                              "$set": enum.to_serializable_dict()})
+
+    def update(self, enum: Enum):
+        collection = self._database[self._enum_collection]
+        collection.update_one({"name": enum.name, "use_case": enum.use_case, "table": enum.table}, {
+                              "$set": enum.to_serializable_dict()})
+
+    def delete(self, enum: Enum):
+        collection = self._database[self._enum_collection]
+        collection.delete_many(
+            {"name": enum.name, "use_case": enum.use_case, "table": enum.table})

src/participation-hub/business-logic-microservice/app/main.py

@@ -13,6 +13,7 @@ import connexion
 from security import swagger_util
 from env_info import is_running_locally, get_resources_path
 from flask import request
+from flask_cors import CORS
 from flask import redirect
 from flask_cors import CORS
 

src/participation-hub/business-logic-microservice/app/routes/enum.py

+#global imports
+from db.entities.enum import Enum
+from db.entities.layer_adapter import LayerAdapter
+from db.repository import Repository
+from db.table_repository import TableRepository
+from db.use_case_repository import UseCaseRepository
+from db.enum_repository import EnumRepository
+
+import json
+from flask import Response, request
+
+table_repository = TableRepository()
+use_case_repository = UseCaseRepository()
+enum_repository = EnumRepository()
+
+def all():
+    return [enum.to_serializable_dict() for enum in enum_repository.all()]
+
+def get_all_for_use_case(use_case: str):
+    '''
+    get all enums assigned to the given use_case
+    '''
+    use_case_repository.put(use_case)
+    return [enum.to_serializable_dict() for enum in enum_repository.all_for_use_case(use_case)]
+
+def one(use_case: str, table: str, name: str):
+    '''
+    fetch a single enum from the DB
+
+    @params:
+        use_case    - Required : String-identifier for the Use-Case the Enum belongs to
+        table       - Required : unique identifier of the Table the Enum belongs to
+        name        - Required : unique identifier for the Enum
+    '''
+    enums = enum_repository.all_for_use_case_and_table_and_name(name, use_case, table)
+
+    return Response(status=200, response=json.dumps([enum.to_serializable_dict() for enum in enums]))
+
+def put_new(use_case: str, table: str, name: str, value: str):
+    '''
+    Put a new Enum to the DB. If the value for this enum already exists, nothing happens.
+
+    @params:
+        use_case    - Required : String-identifier for the Use-Case the Enum belongs to
+        table       - Required : unique identifier of the Table the Enum belongs to
+        name        - Required : unique identifier for the Enum
+    '''
+    existing_enums = enum_repository.all_for_use_case_and_table_and_name(name, use_case, table)
+    enum_target = None
+
+    found = False
+    for enum in existing_enums:
+        if enum.value == value:
+            enum_target = enum
+            break
+
+    if enum_target == None:
+        enum_target = Enum(use_case, table, name, value, len(existing_enums))
+        enum_repository.add(enum_target)
+
+    return Response(status=200, response=json.dumps(enum_target.to_serializable_dict()))
\ No newline at end of file

src/transaction-hub-in/trace-retrieval-microservice/app/messaging/rest_fetcher.py

@@ -2,23 +2,48 @@ from security.token_manager import TokenManager
 import network_constants
 
 from typing import List
-import json, requests
+import json
+import requests
+
 
 class RestFetcher:
+
+    def fetch_enum_value(self, use_case: str, table: str, enum_name: str, enum_value: str) -> int:
+        jwt_token = TokenManager.getInstance().getToken()
+        url = f'https://{network_constants.BUSINESS_LOGIC_HOSTNAME}:{network_constants.BUSINESS_LOGIC_REST_PORT}/api/use-cases/{use_case}/tables/{table}/enum/{enum_name}?value={enum_value}'
+        print(f"calling {url}")
+
+        # query tables for use-case
+        url = url
+        response = requests.put(
+            url,
+            verify=False,
+            proxies={"http": None, "https": None},
+            headers={"Authorization": f"Bearer {jwt_token}"},
+        )
+
+        if response.status_code != 200:
+            raise ValueError(
+                f"Error while retrieving Enum information.\tStatus-code:{response.status_code}")
+
+        enum = json.loads(response.text)
+        return enum["index"]
+
     def fetch_schema_information(self, use_case: str) -> List:
         jwt_token = TokenManager.getInstance().getToken()
-        
+
         # query tables for use-case
         url = f'https://{network_constants.BUSINESS_LOGIC_HOSTNAME}:{network_constants.BUSINESS_LOGIC_REST_PORT}/api/use-cases/{use_case}/tables'
         response = requests.get(
-                url,
-                verify = False, 
-                proxies = { "http":None, "https":None },
-                headers = { "Authorization": f"Bearer {jwt_token}"}
+            url,
+            verify=False,
+            proxies={"http": None, "https": None},
+            headers={"Authorization": f"Bearer {jwt_token}"}
         )
 
         if response.status_code != 200:
-            raise ValueError(f"Error while retrieving schema information.\tStatus-code:{response.status_code}")
+            raise ValueError(
+                f"Error while retrieving schema information.\tStatus-code:{response.status_code}")
 
         tables = json.loads(response.text)
         return tables