Merge branch 'develop' into feature/community-prediction

.gitignore

@@ -17,3 +17,5 @@ src/modules/security/regular_user_credentials.json
 src/modules/security/default_users.json
 
 resources/
+
+reports/

bin/testing.py

@@ -2,7 +2,8 @@ import os
 import sys
 import importlib.util
 import pathlib
-
+import shutil
+import re
 '''
 This script searches for all 'tests/' directories and executes all tests 
 by cd'ing into the dir and executing unittest discover.
@@ -12,29 +13,107 @@ Use command line argument '-w' to run on windows.
 
 PY = sys.argv[2] if (len(sys.argv) > 1 and sys.argv[1] == '-py') else 'python3.7' # use -py to use your own python command
 ROOT = pathlib.Path(__file__).parent.parent.absolute()
+REPORTS = ROOT / 'reports'
 TESTS_FOLDER_NAME = os.path.normpath("/tests")
 
+print("Creating VENV")
+os.system(f"{PY} -m venv venv")
+
+PY = f"~/smart/venv/bin/{PY}"
+
 print("\nSearching for tests at the path: "+ str(ROOT))
 
 count = 0
 resultCodeList = []
+microservice_coverage_paths_set = set()
 for (dirname, dirs, files) in os.walk(ROOT):
 
     #I assume all the tests are placed in a folder named "tests"
     if (TESTS_FOLDER_NAME in str(dirname)) \
         and 'src' in str(dirname) \
         and not(f"{TESTS_FOLDER_NAME}{os.path.normpath('/')}" in str(dirname)) \
-        and not("venv" in str(dirname)):
+        and not("venv" in str(dirname)) \
+        and not("Lib" in str(dirname)):
         try:
             print(f"Executing tests in {dirname}")
             os.chdir(os.path.normpath(dirname))
             # TODO do this during docker image setup
             exit_val = os.system(f"{PY} -m pip install -r ../requirements.txt") # install pip dependencies
-            exit_val = os.system(f"{PY} -m unittest discover") # execute the tests
+            #resultCodeList.append(exit_val)
+            #exit_val = os.system(f"{PY} -m unittest discover") # execute the tests
+            exit_val = os.system(f"{PY} -m coverage run --append --omit=*/site-packages*,*/dist-packages* -m unittest discover") #TEST CODE COVERAGE
+            microservice_coverage_paths_set.add(os.path.normpath(dirname))
+            
             resultCodeList.append(exit_val) #once per folder  i.e if 3 tests are in a folder and crash, there will be just one exit val
         except Exception as e:
             print(e)
             continue
+        try:
+            cur_dir = pathlib.Path(os.path.normpath(dirname)).parent.absolute()
+            filename_regular_expresion = re.compile('(test_.*)|(TEST_.*)')
+            
+
+            for filename in os.listdir(cur_dir):
+                if filename_regular_expresion.match(filename): 
+                    #gets here only if there is a test file which matches the regular expression in the app folder,
+                    
+                    #cur_dir = os.path(dirname).parent()
+                    os.chdir(cur_dir)
+                    print(f"Executing coverage test in {cur_dir}")
+                    exit_val = os.system(f"{PY} -m coverage run --append --omit=*/site-packages* -m unittest discover")
+                    microservice_coverage_paths_set.add(os.path.normpath(cur_dir))
+
+        except Exception as e:
+            print(e)
+            continue
+        
+
+
+#CHANGE FOLDER TO REPORTS, in order to combine the coverage
+try:
+    if not os.path.exists(REPORTS):
+        os.makedirs(REPORTS) 
+except:
+    pass
+
+
+try: 
+    os.chdir(REPORTS)
+    target = REPORTS
+    target = os.path.normpath( str(target) + f'/.coverage' )
+    os.remove(target) #Try to Remove old coverage file, if exists
+except Exception as e:
+    pass
+
+print("Combinging coverages")
+counter = 0
+for path in microservice_coverage_paths_set:
+    try:
+        path += '/.coverage'
+        original = os.path.normpath( path )
+        target = REPORTS
+        target = os.path.normpath( str(target) + f'/.coverage.{counter}' )
+        counter += 1
+
+        shutil.copyfile(original,target) #copy new generated coverage files
+        os.remove(original)
+    except Exception as e:
+        print(e)
+        continue
+try:
+    coverage_xml_path = os.path.normpath( str(REPORTS) + '/coverage.xml')
+    os.remove(coverage_xml_path)
+
+    #coverage_html_path = os.path.normpath( str(REPORTS) + '/htmlcov' )
+    #os.rmdir(coverage_html_path)
+except Exception as e:
+    print(e)
+
+
+print("Generating Combined report")
+os.system(f"{PY} -m coverage combine")
+os.system(f"{PY} -m coverage xml")
+os.system(f"{PY} -m coverage html") #if you want to generate the html as well
 
 firstError = -1
 i = 0
@@ -48,4 +127,5 @@ while i < len(resultCodeList):
 if(firstError<0): #no errors found
     sys.exit(0)
 else:
-    sys.exit(1) #return code>0
+    sys.exit(1) #return code>0
\ No newline at end of file
+

documentation/external_access.md

@@ -31,7 +31,7 @@ This token is used for authentication as _regular user_ on all microservices cur
 adds a blockchain transaction entry for ApplicationType with all the keys and values. These will be converted and stored in our own format for creating multilayers and communities.
 
 # Business Logic Microservice
-https://articonf1.itec.aau.at:30420/api/ui
+https://articonf1.itec.aau.at:30420/api/ui/
 
 This microservice contains use-case specific informations, like schemas and contexts.
 
@@ -41,16 +41,54 @@ This microservice contains use-case specific informations, like schemas and cont
 ## Context information
 ```GET https://articonf1.itec.aau.at:30420/api/use-cases/{use-case}/layers``` returns all layers from the schema used for clustering interally.
 
+# Trace Retrieval Microservice
+https://articonf1.itec.aau.at:30001/api/ui/
+
+This microservice contains the nodes from the transactions preprocessed as defined in *Schema Information*.
+
+```GET https://articonf1.itec.aau.at:30001/api/use_cases/{use_case}/transactions``` returns all flattened transactions, before splitting them into layers.
+
 # Semantic Linking Microservice
 https://articonf1.itec.aau.at:30101/api/ui/
 
-This microservice contains the nodes from the transactions preprocessed as defined in *Schema Information*. Additionally it splits the raw input into multipe layers.
+This microservice splits the preprocessed transactions into multipe layers, calling the splitted transaction per layer nodes.
 
-```GET https://articonf1.itec.aau.at:30101/api/use-cases/{use-case}/nodes``` returns all preprocessed transactions, called nodes, before splitting them into layers.
+```GET https://articonf1.itec.aau.at:30101/api/use-cases/{use_case}/tables/{table_name}/layers/{layer_name}/nodes ``` returns all splitted transactions, called nodes, for the layer layer_name.
 
 # Role Stage Discovery Microservice
-https://articonf1.itec.aau.at:30103/api/ui
+https://articonf1.itec.aau.at:30103/api/ui/
+
+This microservice contains the communities based on clusters and similarities between communities. It additionally contains time slices with subsets of clusters, which's transaction happened in the corresponding time window. 
+Schemas and Input data are supplied by the [Business Logic microservice](https://articonf1.itec.aau.at:30420/api/ui), [Semantic Linking microservice](https://articonf1.itec.aau.at:30101/api/ui/) and [Trace Retrieval microservice](https://articonf1.itec.aau.at:30101/api/ui/).
+
+## Layers
+Contains information about the schema copied from the Business Logic microservice.
+Returns the Schemas and/or Input data used for calculating the clustering which is further used for calculating the similarity.
+
+```GET https://articonf1.itec.aau.at:30103/api/use-cases/{use_case}/layers``` returns layer infos for the given use-case.
+
+```GET https://articonf1.itec.aau.at:30103/api/use-cases/{use_case}/tables/{table}/layers/{layer_name}``` returns the layer information for only the one layer.
+
+```GET https://articonf1.itec.aau.at:30103/api/use-cases/{use_case}/tables/{table}/layers/{layer_name}/nodes``` contains all the nodes contained in the layer fetched from the Semantic Linking microservice.
+
+## Clusters
+Contains the clustering results. Clustering is performed on all nodes inside one layer. Furthermore the clusters are partitioned based on timestamps.
+
+```GET https://articonf1.itec.aau.at:30103/api/use-cases/{use_case}/tables/{table}/layers/{layer_name}/clusters``` returns the identified clusters.
+
+```GET https://articonf1.itec.aau.at:30103/api/use-cases/{use_case}/tables/{table}/layers/{layer_name}/timeslices``` returns the identified clusters partitioned based on their nodes' timestamps.
+
+## RunId
+When a similarity computation is executed, it has an associated RunId which is used to uniquely identify that execution.
+
+```GET https://articonf1.itec.aau.at:30103/api/runIds``` returns all RunIds in the db.
+
+## Similarity
+Returns the computed similarity. Two clusters belonging to the SAME layer will be given a similarity value by comparing them to another cluster belonging to a DIFFERENT layer. This is done for every cluster in the input data. This querry returns all the calculated similarity values, given the criteria (i.e belonging to a use-case,table etc).
+
+```GET https://articonf1.itec.aau.at:30103/api/use_cases/{use_case}/tables/{table}/clusterSimilarity``` returns all similarity values for the given use-case and table.
 
-This microservice contains the communities based on clusters and similarities between communities. It additionally contains time slices with subsets of clusters, which's transaction happened in the corresponding time.
+## Connected Cluster
+Intermediary data-structure used only by the function which computes the similarity. Clusters are connected only to other clusters belonging to a DIFFERENT layer.
 
-The endpoints are currently refactored, so please check the Swagger UI autogenerated documentation on its website.
+```GET https://articonf1.itec.aau.at:30103/api/use_cases/{use_case}/tables{table}/connectedClusters``` returns all connected clusters for the given use-case and table.
\ No newline at end of file

src/data-hub/role-stage-discovery-microservice/app/db/repository.py

@@ -130,7 +130,49 @@ class Repository(MongoRepositoryBase):
         if (run_id == None):
             entries = super().get_entries(self._connected_clusters_collection, projection={'_id': 0})
         else:
-            entries = super().get_entries(self._similarity_collection, selection={'cluster_runId' : run_id}, projection={'_id': 0})
+            entries = super().get_entries(self._connected_clusters_collection, selection={'cluster_runId' : run_id}, projection={'_id': 0})
+        
+        output = []
+        for ent in entries:
+            output.append(ent)
+        return output
+        # print(ent)
+        #return [Cluster(cluster_dict=e, from_db=True) for e in entries]
+
+    def get_connected_clusters_for_use_case(self,use_case, run_id: str=None):#, layer_name: str):
+        ''' Get Connected Clusters Data given the Use-Case from DB '''
+        if (run_id == None):
+            entries = super().get_entries(self._connected_clusters_collection, selection={'cluster_use_case': use_case}, projection={'_id': 0})
+        else:
+            entries = super().get_entries(self._connected_clusters_collection, selection={'cluster_runId' : run_id, 'cluster_use_case': use_case}, projection={'_id': 0})
+        
+        output = []
+        for ent in entries:
+            output.append(ent)
+        return output
+        # print(ent)
+        #return [Cluster(cluster_dict=e, from_db=True) for e in entries]
+
+    def get_connected_clusters_for_table(self,use_case,table, run_id: str=None):#, layer_name: str):
+        ''' Get Connected Clusters Data given the Use-Case and Table from DB '''
+        if (run_id == None):
+            entries = super().get_entries(self._connected_clusters_collection, selection={'cluster_use_case': use_case,'cluster_table': table}, projection={'_id': 0})
+        else:
+            entries = super().get_entries(self._connected_clusters_collection, selection={'cluster_runId' : run_id,'cluster_use_case': use_case,'cluster_table': table}, projection={'_id': 0})
+        
+        output = []
+        for ent in entries:
+            output.append(ent)
+        return output
+        # print(ent)
+        #return [Cluster(cluster_dict=e, from_db=True) for e in entries]
+
+    def get_connected_clusters_by_name(self,use_case, table, layer_name, run_id: str=None):#, layer_name: str):
+        ''' Get Connected Clusters Data from DB '''
+        if (run_id == None):
+            entries = super().get_entries(self._connected_clusters_collection, selection={'cluster_use_case': use_case,'cluster_table': table, 'cluster_layer' : layer_name}, projection={'_id': 0})
+        else:
+            entries = super().get_entries(self._connected_clusters_collection, selection={'cluster_runId' : run_id,'cluster_use_case': use_case,'cluster_table': table, 'cluster_layer' : layer_name}, projection={'_id': 0})
         
         output = []
         for ent in entries:
@@ -175,8 +217,38 @@ class Repository(MongoRepositoryBase):
             output.append(e)
         return output
         """
+    def get_similarity_use_case(self,skipNr,batchSize,use_case, run_id: str=None):
+        ''' Get Similarity Data from DB '''
+        
+        if (run_id == None):
+            entries = super().get_entries(self._similarity_collection, selection={'use_case' : use_case}, projection={'_id': 0})
+        else:
+            entries = super().get_entries(self._similarity_collection, selection={'use_case' : use_case, 'runId' : run_id}, projection={'_id': 0})
+            
+        #
+        return list(entries.sort([('_id', -1)]).skip(skipNr).limit(batchSize))
+    
+    def get_similarity_table(self,skipNr,batchSize,use_case,table, run_id: str=None):
+        ''' Get Similarity Data from DB '''
+        
+        if (run_id == None):
+            entries = super().get_entries(self._similarity_collection, selection={'use_case' : use_case, 'table': table}, projection={'_id': 0})
+        else:
+            entries = super().get_entries(self._similarity_collection, selection={'use_case' : use_case, 'table': table, 'runId' : run_id}, projection={'_id': 0})
+            
+        #
+        return list(entries.sort([('_id', -1)]).skip(skipNr).limit(batchSize))
 
-
+    def get_similarity_layer(self,skipNr,batchSize,use_case,table,layer, run_id: str=None):
+        ''' Get Similarity Data from DB '''
+        
+        if (run_id == None):
+            entries = super().get_entries(self._similarity_collection, selection={'use_case' : use_case, 'table': table, 'cluster_layer' : layer}, projection={'_id': 0})
+        else:
+            entries = super().get_entries(self._similarity_collection, selection={'use_case' : use_case, 'table': table, 'cluster_layer' : layer, 'runId' : run_id}, projection={'_id': 0})
+            
+        #
+        return list(entries.sort([('_id', -1)]).skip(skipNr).limit(batchSize))
 #endregion
 
 #region connected_run

src/data-hub/role-stage-discovery-microservice/app/main.py

@@ -18,9 +18,11 @@ from pathlib import Path
 from env_info import is_running_locally, get_resources_path
 from flask import request
 from flask import redirect
+from flask_cors import CORS
 
 # load swagger config 
 app = connexion.App(__name__, specification_dir='configs/')
+CORS(app.app)
 
 @app.app.before_request
 def before_request():

src/data-hub/role-stage-discovery-microservice/app/processing/similarityFiles/dataInput.py

@@ -67,7 +67,7 @@ def loadJson(url:str) :
     return jsonData
 
 
-def getClusterDataFromMongo(layerNameList,limitNrCluster,limitNrNodes):
+def getClusterDataFromMongo(layerNameList,limitNrCluster,limitNrNodes,use_case,table):
     ''' Calculates the nr of connections/weights between the clusters contained in the "inputLayerDict". Connections are made between clusters from DIFFERENT layers.
 
     :param List[string] layerNameList: Name of the layers to pull from the DB
@@ -93,7 +93,7 @@ def getClusterDataFromMongo(layerNameList,limitNrCluster,limitNrNodes):
     #imports and translates the data from JSON into usefull format
     #returns layerdiction -> Layer -> clusterDict -> Cluster -> nodesDict -> Nodes    
     for name in layerNameList:
-        newData = get_mongoDB_cluster_by_layerName(name)#repo.get_clusters_for_layer(name)
+        newData = get_mongoDB_cluster_by_layerName(use_case,table,name)#repo.get_clusters_for_layer(name)
         if newData is not None and len(newData) != 0:
             layerDict = populateWithNewNodesSingleLayer(newData[0:limitNrCluster],layerDict,limitNrNodes)
 
@@ -290,7 +290,7 @@ def makeChangeNodesDict(inputList,cluster_label,cluster_layer):
         outputDict[key]= newNode
     return outputDict
         
-def get_mongoDB_cluster_by_layerName(name):
-    res = repo.get_clusters_for_layer(name)
+def get_mongoDB_cluster_by_layerName(use_case, table , layer_name):
+    res = repo.get_clusters_for_layer(use_case, table, layer_name)
     return [c.to_serializable_dict() for c in res]
 

src/data-hub/role-stage-discovery-microservice/app/processing/similarityFiles/dataOutput.py

@@ -6,7 +6,7 @@ from processing.similarityFiles.miscFunctions import *
 from db.repository import Repository
 repo = Repository()
 
-def outputFileLayerFunction(layerDict,limitNrNodes,limitNrCluster,runId):
+def outputFileLayerFunction(layerDict,limitNrNodes,limitNrCluster,runId,table,use_case):
     ''' Writes the layerDict data to a JSON file.
 
     :param Dict{string: Layer} layerDict: Object which contains Data about the Layers, Clusters and Nodes
@@ -17,7 +17,7 @@ def outputFileLayerFunction(layerDict,limitNrNodes,limitNrCluster,runId):
 
     '''  
 
-    layerJSON = convertLayerDictToJSON(layerDict,runId)
+    layerJSON = convertLayerDictToJSON(layerDict,runId,table,use_case)
     outputJSON = json.dumps(layerJSON, default=lambda o: o.__dict__, indent=4)
             
     try:
@@ -28,7 +28,7 @@ def outputFileLayerFunction(layerDict,limitNrNodes,limitNrCluster,runId):
 
 
 
-def outputFileSimilFunction(similarityDict,limitNrNodes,limitNrCluster,runId):
+def outputFileSimilFunction(similarityDict,limitNrNodes,limitNrCluster,runId,table,use_case):
 
     ''' Writes the similarityDict data to a JSON file.
 
@@ -40,7 +40,7 @@ def outputFileSimilFunction(similarityDict,limitNrNodes,limitNrCluster,runId):
 
     '''  
     
-    similJSON = convertSimilarityDictToJSON(similarityDict,runId)
+    similJSON = convertSimilarityDictToJSON(similarityDict,runId,table,use_case)
     outputJSON = json.dumps(similJSON, default=lambda o: o.__dict__, indent=4)
 
     try: 
@@ -77,7 +77,7 @@ def outputFileTimeFunction(timelist,limitNrNodes,limitNrCluster,runId):
         print("Error occured when writing the resultTimeExec file")
 
 
-def outputMongoConnClustDict(inputDict,runId):
+def outputMongoConnClustDict(inputDict,runId,table,use_case):
 
     ''' Stores connected_clusters in the database.
 
@@ -89,9 +89,9 @@ def outputMongoConnClustDict(inputDict,runId):
 
     #inputDict["Timestamp"] = str(datetime.datetime.now())
     
-    add_conn_clusters(inputDict,runId)
+    add_conn_clusters(inputDict,runId,table,use_case)
 
-def outputMongoSimilarity(inputDict,runId):
+def outputMongoSimilarity(inputDict,runId,table,use_case):
     ''' Stores cluster_similarity in the database.
 
     :param Dict() inputDict: Contains the data to insert
@@ -99,7 +99,7 @@ def outputMongoSimilarity(inputDict,runId):
     :param string runId: Id of the Run 
 
     ''' 
-    add_similarity(inputDict,runId)
+    add_similarity(inputDict,runId,table,use_case)
 
 def add_connected_run():
 
@@ -116,7 +116,7 @@ def add_connected_run():
     inserted_result = repo.add_connected_run(runDict)
     return str(inserted_result.inserted_id)
 
-def add_conn_clusters(inputDict,runId):
+def add_conn_clusters(inputDict,runId,table,use_case):
     ''' Stores connected_clusters in the database.
 
     :param Dict() inputDict: Contains the data to insert
@@ -125,11 +125,11 @@ def add_conn_clusters(inputDict,runId):
 
     ''' 
     
-    outputJSON = convertLayerDictToJSON(inputDict,runId)
+    outputJSON = convertLayerDictToJSON(inputDict,runId,table,use_case)
     for element in outputJSON:
         repo.add_connected_cluster(element)
 
-def add_similarity(inputDict,runId):
+def add_similarity(inputDict,runId,table,use_case):
     ''' Stores cluster_similarity in the database.
 
     :param Dict() inputDict: Contains the data to insert
@@ -138,6 +138,6 @@ def add_similarity(inputDict,runId):
 
     ''' 
 
-    outputJSON = convertSimilarityDictToJSON(inputDict,runId)
+    outputJSON = convertSimilarityDictToJSON(inputDict,runId,table,use_case)
     for element in outputJSON:
         repo.add_single_similarity(element)
\ No newline at end of file

src/data-hub/role-stage-discovery-microservice/app/processing/similarityFiles/miscFunctions.py

@@ -42,7 +42,7 @@ def totalNumberOfClusters(inputLayerDict):
                 
     return clustCount
 
-def convertLayerDictToJSON(layerDict, runId):
+def convertLayerDictToJSON(layerDict, runId,table,use_case):
     ''' Converts a Layer object to JSON format.
 
     :param Dict{string: Layer} layerDict: Object which contains Data about the Layers, Clusters and Nodes
@@ -56,7 +56,9 @@ def convertLayerDictToJSON(layerDict, runId):
         for curCluster in curLayer.cluster_Dict.values():
             outputJSON.append({
                 "cluster_label" : curCluster.cluster_label,
-                "cluster_layer" : curCluster.cluster_layer, 
+                "cluster_layer" : curCluster.cluster_layer,
+                "cluster_table" : table,
+                "cluster_use_case": use_case,
                 "cluster_runId" : runId,
                 "cluster_connClustDict" : changeTupleDictToDictList(curCluster.cluster_connClustDict),
                 "cluster_connNodesDict" : getFrozensetFromConnNodesDict(curCluster.cluster_connNodesDict), #Don
@@ -109,7 +111,7 @@ def getFrozensetFromConnNodesDict(inputDict):
     return output
 
 
-def convertSimilarityDictToJSON(inputDict,runId):
+def convertSimilarityDictToJSON(inputDict,runId,table,use_case):
     ''' Converts a Similarity Dictionary to JSON format. For outputting to DB
 
     :param Dict{} similarityDict: Object which contains Data about the Computed similarities between Clusters
@@ -125,6 +127,8 @@ def convertSimilarityDictToJSON(inputDict,runId):
         auxDict["cluster_layer"] = tupleKey[2]  
         auxDict["similarityValues"] = inputDict[tupleKey]    
         auxDict["runId"] = runId
+        auxDict["table"] = table
+        auxDict["use_case"] = use_case
         similList.append(auxDict)
     similToJSON = similList
     #outputJSON = json.dumps(similToJSON, default=lambda o: o.__dict__, indent=4)

src/data-hub/role-stage-discovery-microservice/app/processing/similarityFiles/similarityMain.py

@@ -39,7 +39,7 @@ from processing.similarityFiles.dataOutput import *
 
 outputToFileFLAG = True
 
-def main(layerNameList:List[str] = ["Price_Layer","FinishedTime_Layer","Destination_Layer"]):   
+def main(layerNameList:List[str] , table:str , use_case: str):   
     '''
     Executes the similarity calculation by calculating weights between clusters in different layers.
     Then calculating the Euclidean distance between nodes in the same layer based on one other layer each.
@@ -48,7 +48,8 @@ def main(layerNameList:List[str] = ["Price_Layer","FinishedTime_Layer","Destinat
     :param layerNameList: The list of layer names as strings
     '''
     print("Entered Similarity Main")
-
+    if len(layerNameList)==0:
+        return
     timelist = []
     timelist.append(currentTime())#starting time
     
@@ -67,7 +68,7 @@ def main(layerNameList:List[str] = ["Price_Layer","FinishedTime_Layer","Destinat
     limitNrNodes = -1 #per Layer
 
 
-    layerDict = getClusterDataFromMongo(layerNameList,limitNrCluster,limitNrNodes)
+    layerDict = getClusterDataFromMongo(layerNameList,limitNrCluster,limitNrNodes,use_case,table)
     if layerDict is None or len(layerDict) == 0:
         LOGGER.error(f"No data for any of the following layers existed: {str(layerNameList)}. Similarity calculation was not performed.")
         return
@@ -98,13 +99,13 @@ def main(layerNameList:List[str] = ["Price_Layer","FinishedTime_Layer","Destinat
     
     if (outputToFileFLAG == True):
         print("Outputing data")
-        outputFileLayerFunction(layerDict,totalNodes,totalClusters,runId)
-        outputFileSimilFunction(similarityDict,totalNodes,totalClusters,runId)
+        outputFileLayerFunction(layerDict,totalNodes,totalClusters,runId,table,use_case)
+        outputFileSimilFunction(similarityDict,totalNodes,totalClusters,runId,table,use_case)
         outputFileTimeFunction(timelist,totalNodes,totalClusters,runId)
 
     #Output to DB
-    outputMongoConnClustDict(layerDict,runId)
-    outputMongoSimilarity(similarityDict,runId)
+    outputMongoConnClustDict(layerDict,runId,table,use_case)
+    outputMongoSimilarity(similarityDict,runId,table,use_case)
 
 
     #Currently not used in the calculation of connections/similarity, developed for possible future uses
@@ -122,6 +123,6 @@ def main(layerNameList:List[str] = ["Price_Layer","FinishedTime_Layer","Destinat
     return
 
 ##########START##########
-if __name__ is '__main__':
-    main()
+#if __name__ is '__main__':
+    #main()
 #########FINISH##########

src/data-hub/role-stage-discovery-microservice/app/requirements.txt

@@ -8,11 +8,13 @@ Click==7.0
 clickclick==1.2.2
 colorama==0.4.3
 connexion==2.6.0
+coverage==5.3.1
 cryptography==3.1
 cycler==0.10.0
 decorator==4.4.1
 Deprecated==1.2.7
 Flask==1.1.1
+Flask-Cors==3.0.10
 idna==2.8
 importlib-metadata==1.5.0
 inflection==0.3.1

src/data-hub/role-stage-discovery-microservice/app/routes/clustersets.py

@@ -4,8 +4,8 @@ from db.entities import ClusterSet
 
 repo = Repository()
 
-def get_by_name(use_case, use_case_table, name):
-    res = repo.get_clusters_for_layer(use_case, use_case_table, name)
+def get_by_name(use_case, table, layer_name):
+    res = repo.get_clusters_for_layer(use_case, table, layer_name)
     if res is None or len(res) == 0:
         return Response(status=404)
     else:

src/data-hub/role-stage-discovery-microservice/app/routes/connClusters.py

@@ -16,3 +16,45 @@ def get_conn_clusters():
     else:
 
         return result
+
+def get_conn_clusters_use_case(use_case):
+    ''' Gets connected_clusters from the database.
+
+        :returns: Returns similarity objects from the DB
+        :rtype: Dict
+    '''  
+    result = repo.get_connected_clusters_for_use_case(use_case)
+    if result is None or len(result) == 0:        
+        print("MongoDb Get Error: Response 404")
+        return Response(status=404)
+    else:
+
+        return result
+
+def get_conn_clusters_table(use_case,table):
+    ''' Gets connected_clusters from the database.
+
+        :returns: Returns similarity objects from the DB
+        :rtype: Dict
+    '''  
+    result = repo.get_connected_clusters_for_table(use_case, table)
+    if result is None or len(result) == 0:        
+        print("MongoDb Get Error: Response 404")
+        return Response(status=404)
+    else:
+
+        return result
+
+def get_conn_clusters_name(use_case,table,layer_name):
+    ''' Gets connected_clusters from the database.
+
+        :returns: Returns similarity objects from the DB
+        :rtype: Dict
+    '''  
+    result = repo.get_connected_clusters_by_name(use_case,table,layer_name)
+    if result is None or len(result) == 0:        
+        print("MongoDb Get Error: Response 404")
+        return Response(status=404)
+    else:
+
+        return result

src/data-hub/role-stage-discovery-microservice/app/routes/layers.py

@@ -26,15 +26,15 @@ def get_by_use_case(use_case):
     else:
         return Response(status=404)
 
-def get_by_table(use_case, use_case_table):
-    res = repo.get_layers_for_table(use_case, use_case_table)
+def get_by_table(use_case, table):
+    res = repo.get_layers_for_table(use_case, table)
     if len(res) > 0:
         return [l.to_serializable_dict() for l in res]
     else:
         return Response(status=404)
 
-def get_by_name(use_case, use_case_table, name):
-    res = repo.get_layer_by_name(use_case, use_case_table, name)
+def get_by_name(use_case, table, layer_name):
+    res = repo.get_layer_by_name(use_case, table, layer_name)
     if res is not None:
         return res.to_serializable_dict() 
     else:
@@ -43,8 +43,8 @@ def get_by_name(use_case, use_case_table, name):
 #endregion
 
 #region nodes
-def get_nodes(use_case, use_case_table, name):
-    res = repo.get_layer_nodes(use_case, use_case_table, name)
+def get_nodes(use_case, table, layer_name):
+    res = repo.get_layer_nodes(use_case, table, layer_name)
     # print(res)
     return res
 

src/data-hub/role-stage-discovery-microservice/app/routes/similarity.py

@@ -23,3 +23,60 @@ def get_similarity(layer_name,batchNr):
         return Response(status=404)
     else:
         return result
+
+def get_similarity_use_case(use_case,batchNr): 
+    ''' Gets cluster_similarity from the database.
+
+        :returns: Returns similarity objects from the DB
+        :rtype: Dict
+    '''  
+    batchSize = 1000
+    if int(batchNr)<0:
+        print("Batch number needs to be a positive integer")
+        return Response(status=404)
+    skipNr = batchSize*int(batchNr)
+    #get_similarity(self,skipNr,batchSize, cluster_layer: str= None, run_id: str=None)
+    result = repo.get_similarity_use_case(skipNr, batchSize, use_case)
+    if result is None or len(result) == 0:        
+        print("MongoDb Get Error: Response 404")
+        return Response(status=404)
+    else:
+        return result
+
+def get_similarity_table(use_case,table,batchNr): 
+    ''' Gets cluster_similarity from the database.
+
+        :returns: Returns similarity objects from the DB
+        :rtype: Dict
+    '''  
+    batchSize = 1000
+    if int(batchNr)<0:
+        print("Batch number needs to be a positive integer")
+        return Response(status=404)
+    skipNr = batchSize*int(batchNr)
+    #get_similarity(self,skipNr,batchSize, cluster_layer: str= None, run_id: str=None)
+    result = repo.get_similarity_table(skipNr, batchSize, use_case,table)
+    if result is None or len(result) == 0:        
+        print("MongoDb Get Error: Response 404")
+        return Response(status=404)
+    else:
+        return result
+
+def get_similarity_layer(use_case,table,layer_name,batchNr): 
+    ''' Gets cluster_similarity from the database.
+
+        :returns: Returns similarity objects from the DB
+        :rtype: Dict
+    '''  
+    batchSize = 1000
+    if int(batchNr)<0:
+        print("Batch number needs to be a positive integer")
+        return Response(status=404)
+    skipNr = batchSize*int(batchNr)
+    #get_similarity(self,skipNr,batchSize, cluster_layer: str= None, run_id: str=None)
+    result = repo.get_similarity_layer(skipNr, batchSize,use_case,table, layer_name)
+    if result is None or len(result) == 0:        
+        print("MongoDb Get Error: Response 404")
+        return Response(status=404)
+    else:
+        return result

src/data-hub/role-stage-discovery-microservice/app/routes/timeslices.py

@@ -4,8 +4,8 @@ from db.entities import TimeSlice
 
 repo = Repository()
 
-def get_by_name(use_case, use_case_table, name):
-    res = repo.get_time_slices_by_name(use_case, use_case_table, name)
+def get_by_name(use_case, table, layer_name):
+    res = repo.get_time_slices_by_name(use_case, table, layer_name)
     
     if res is not None and len(res) != 0:
         return [e.to_serializable_dict() for e in res]

src/data-hub/role-stage-discovery-microservice/app/run_similarity_calc.py

@@ -7,17 +7,38 @@ repo = Repository()
 def run_similarity_calc_per_use_case():
     layers = repo.get_layers()
 
-    uc_layers = {}
+    # uc_layers = {}
+    # for layer in layers:
+    #     uc = layer.use_case
+    #     if uc not in uc_layers:
+    #         uc_layers[uc] = []
+    #     uc_layers[uc].append(layer.layer_name)
+
+    # for key in uc_layers:
+    #     layers2 = uc_layers[key]
+    #     print(f"Running for use case {key} with layers {str(layers2)}.")
+    #     SimilarityCalc.main(layerNameList=layers2)
+
+    uc_dict = dict()
+    # use_case[table[layer_name]]
     for layer in layers:
-        uc = layer.use_case
-        if uc not in uc_layers:
-            uc_layers[uc] = []
-        uc_layers[uc].append(layer.layer_name)
-
-    for key in uc_layers:
-        layers = uc_layers[key]
-        print(f"Running for use case {key} with layers {str(layers)}.")
-        SimilarityCalc.main(layerNameList=layers)
+        use_case = layer.use_case
+        table = layer.use_case_table
+        if use_case not in uc_dict:
+            uc_dict[use_case] = dict()
+
+        #aux = uc_dict[use_case]
+        if table not in uc_dict[use_case]:
+            uc_dict[use_case][table] = []
+
+        uc_dict[use_case][table].append(layer.layer_name)
+
+    for uc in uc_dict:
+        for table in uc_dict[uc]:
+            layers2 = uc_dict[uc][table]
+            print(f"Running for use case {uc}, table {table}, with layers {str(layers2)}.")
+            SimilarityCalc.main(layers2,table,uc)
+
     
 
 if __name__ == '__main__':

src/data-hub/role-stage-discovery-microservice/app/test_coverage.py

+import unittest
+import sys
+for path in ['../', './']:
+    sys.path.insert(1, path)
+
+#####################################
+### Don't include for test report ###
+#####################################
+try: 
+    class TestCoverage(unittest.TestCase):
+        def test_init_main(self):
+            try:
+                # python -m unittest discover
+                from db.entities import Cluster
+
+
+                from datetime import date, datetime
+                import json
+
+                # add modules folder to interpreter path
+                import sys
+                import os
+                modules_path = '../../../modules/'
+                if os.path.exists(modules_path):
+                    sys.path.insert(1, modules_path)
+
+                ### init logging ###
+                import logging
+                LOG_FORMAT = ('%(levelname) -5s %(asctime)s %(name)s:%(funcName) -35s %(lineno) -5d:  %(message)s')
+                logging.basicConfig(level=logging.INFO, format=LOG_FORMAT)
+                LOGGER = logging.getLogger(__name__)
+
+                #############################
+                import connexion
+                from security import swagger_util
+                from pathlib import Path
+                from env_info import is_running_locally, get_resources_path
+                from flask import request
+                from flask import redirect
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+
+            try:
+                import main  #error when importing main, ModuleNotFoundError: No module named 'security'
+            #exec(open('main.py').read())
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+
+        def test_init_run_clustering(self):
+            try:
+                import sys
+                import os
+                modules_path = '../../../modules/'
+                if os.path.exists(modules_path):
+                    sys.path.insert(1, modules_path)
+
+                import json
+                from db.entities import Layer, Cluster
+                from typing import List, Dict, Tuple, Any
+                from db.repository import Repository
+                from processing.clustering import Clusterer, ClusterResult
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+            
+            try:
+                import run_clustering
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+
+        def test_init_run_node(self):
+            try:
+                import sys
+                import os
+                modules_path = '../../../modules/'
+                if os.path.exists(modules_path):
+                    sys.path.insert(1, modules_path)
+
+                import json
+                import urllib3
+                urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+            
+            try:
+                import processing.fetching.fetching as f
+                import run_node_fetching
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+
+        def test_init_run_similarity(self):
+            try:
+                import processing.similarityFiles.similarityMain as SimilarityCalc
+                from db.repository import Repository
+
+                import run_similarity_calc
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+
+        def test_init_run_time(self):
+            try:
+                import sys
+                import os
+                modules_path = '../../../modules/'
+                if os.path.exists(modules_path):
+                    sys.path.insert(1, modules_path)
+
+                import json
+                from datetime import datetime, date
+                from db.repository import Repository
+                from db.entities import ClusterSet, Cluster, Layer, TimeSlice
+                from typing import Tuple, Dict, Any, List
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+
+            try:
+                import run_time_slicing
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+
+
+    if __name__ == '__main__':
+        unittest.main()
+except Exception as e:
+    print ("Exception found:")
+    print (e)
\ No newline at end of file

src/data-hub/role-stage-discovery-microservice/app/tests/test_cluster.py

+
 import unittest
 import sys
 for path in ['../', './']:
@@ -19,6 +20,5 @@ class TestCluster(unittest.TestCase):
         self.assertEqual(1, c.cluster_label)
         self.assertEqual([1, 2, 3], c.nodes)
 
-
 if __name__ == '__main__':
     unittest.main()

src/data-hub/role-stage-discovery-microservice/app/tests/test_cluster_result.py

@@ -11,7 +11,9 @@ class TestClusterResult(unittest.TestCase):
     converter:ClusterResultConverter = None
 
     def setUp(self):
+    
         self.converter = ClusterResultConverter()
+    
 
     def test_result_undefined_feature(self):
         cluster_groups = self._get_some_cluster_groups_1d()
@@ -29,11 +31,13 @@ class TestClusterResult(unittest.TestCase):
             cluster_groups=cluster_groups,
             features=['v']
         )
+        
 
         self.assert_correct_cluster_result_len(cluster_groups, cluster_res)
         self.assert_correct_cluster_result_labels(['-1.0 -- 1.0','10.0 -- 11.0','2.0 -- 2.0'], cluster_res)
 
     def test_result_2d_features(self):
+
         cluster_groups = self._get_some_cluster_groups_2d()
         cluster_res = self.converter.convert_to_cluster_results(
             cluster_groups=cluster_groups,
@@ -42,6 +46,7 @@ class TestClusterResult(unittest.TestCase):
 
         self.assert_correct_cluster_result_len(cluster_groups, cluster_res)
         self.assert_correct_cluster_result_labels([str((0.0,0.0)), str((10.5,10.5)), str((2.0,2.0)), str((3.0,6.0))], cluster_res)
+
     
 
     #region Custom Assertions
@@ -52,6 +57,7 @@ class TestClusterResult(unittest.TestCase):
             self.assertEqual(len(expected[i]), len(actual[i].nodes))
             self.assertEqual(expected[i], actual[i].nodes)
 
+
     def assert_correct_cluster_result_labels(self, expected: List[str], actual: Dict[Any, ClusterResult]):
         self.assertEqual(len(expected), len(actual))
         for i in range(len(expected)):

src/data-hub/role-stage-discovery-microservice/app/tests/test_clusterer.py

+
+
 import unittest
 import sys
 for path in ['../', './']:
     sys.path.insert(1, path)
 
+
 # python -m unittest discover
 from processing.clustering import Clusterer, ClusterResult
 import numpy as np
@@ -187,8 +190,9 @@ class TestClusterer(unittest.TestCase):
                 self.fail(f"Cluster key ({k}, {type(k)}) not in result.")
             
             self.assertListEqual(expected[k], actual[k].nodes)
-   
+
     #endregion helper methods
 
 if __name__ == '__main__':
     unittest.main()
+

src/data-hub/role-stage-discovery-microservice/app/tests/test_similarity.py

@@ -23,7 +23,7 @@ import json
 class TestSimilarity(unittest.TestCase):
     '''Tests the similarity calculation which works without object orientation.'''
 
-    def test_integration_similarityCalculation(self):
+    def test_integration_calculateSimilarity_ClustersDict_CorrectValue(self):
         '''
         Only for testing, can be deleted at any time.\n
         Served as a testing example to make sure the computations are correct

src/data-hub/semantic-linking-microservice/app/dummy_upload/smart_energy/dummy_upload.py

@@ -9,12 +9,14 @@ for modules_path in modules_paths:
         sys.path.insert(1, modules_path)
 
 from messaging.MessageHandler import MessageHandler
+from db.repository import Repository
 
 # file to read the data from
-CSV_FILE = r'Energy_Dataset.csv'
-handler = MessageHandler()
+CSV_FILE = r'dummy_upload\smart_energy\Energy_Dataset.csv'
+handler = MessageHandler(Repository())
 
 
+processed_transactions = []
 def upload_transaction(transaction):
     '''{"type": "new-trace", 
     "content": {"use_case": "smart-energy", "table": "smart-energy", "id": "dd2c5146c919b046d77a32a5cf553d5133163562f7b7e1298c878c575d516025", 
@@ -28,7 +30,31 @@ def upload_transaction(transaction):
         'id': uid,
         'properties': transaction,
     }
-    handler.handle_new_trace(t)
+    # handler.handle_new_trace(t)
+    
+    processed_transactions.append(t)
+
+
+def store_transactions_for_mirsat():
+    '''
+    Stores the processed transactions as if they would be returned 
+    by Trace Retrieval microservice after fixing the message queue bug.
+    '''
+    flattened_transactions = []
+    for transaction in processed_transactions:
+        transaction = transaction['properties']
+
+        transaction['use_case'] = transaction['ApplicationType']
+        del transaction['ApplicationType']
+
+        transaction['table'] = transaction['docType']
+        del transaction['docType']
+
+        flattened_transactions.append(transaction)
+
+    import json
+    with open('flattened_smart_energy_data.json', 'w') as file:
+        file.write(json.dumps(flattened_transactions))
 
 
 if __name__ == '__main__':    
@@ -56,3 +82,4 @@ if __name__ == '__main__':
 
             upload_transaction(transaction)
 
+    store_transactions_for_mirsat()
\ No newline at end of file

src/data-hub/semantic-linking-microservice/app/main.py

@@ -20,6 +20,7 @@ from env_info import is_running_locally, get_resources_path
 from messaging.ReconnectingMessageManager import ReconnectingMessageManager
 from messaging.MessageHandler import MessageHandler
 from flask import request
+from flask_cors import CORS
 from flask import redirect
 
 # init message handler
@@ -30,6 +31,7 @@ def message_received_callback(channel, method, properties, body):
 
 # load swagger config
 app = connexion.App(__name__, specification_dir='configs/')
+CORS(app.app)
 
 @app.app.before_request
 def before_request():

src/data-hub/semantic-linking-microservice/app/messaging/MessageHandler.py

@@ -83,12 +83,13 @@ class MessageHandler:
             return
 
         nodes = []
-
+        
         for layer in layers:
             node = {}
                 
             for prop in layer.total_properties:
                 node[prop] = content["properties"][prop]
+                
 
             node["layer_name"] = layer.layer_name
             node["table"] = layer.table

src/data-hub/semantic-linking-microservice/app/requirements.txt

@@ -7,8 +7,10 @@ click==7.1.2
 clickclick==1.2.2
 colorama==0.4.3
 connexion==2.7.0
+coverage==5.3.1
 cryptography==3.1
 Flask==1.1.2
+Flask-Cors==3.0.10
 idna==2.9
 importlib-metadata==1.6.1
 inflection==0.5.0

src/data-hub/semantic-linking-microservice/app/test_coverage.py

+import unittest
+import sys
+for path in ['../', './']:
+    sys.path.insert(1, path)
+
+#####################################
+### Don't include for test report ###
+#####################################
+try:
+    class TestCoverage(unittest.TestCase):
+        def test_init_main(self):
+            try:
+                # add modules folder to interpreter path
+                import sys
+                import os
+                import prance
+                from pathlib import Path
+                modules_path = '../../../modules/'
+                if os.path.exists(modules_path):
+                    sys.path.insert(1, modules_path)
+
+                # init logging to file
+                import logging
+                LOG_FORMAT = ('%(levelname) -5s %(asctime)s %(name)s:%(funcName) -35s %(lineno) -5d:  %(message)s')
+                logging.basicConfig(level=logging.INFO, format=LOG_FORMAT)
+                LOGGER = logging.getLogger(__name__)
+
+                #################################
+                import connexion
+                from security import swagger_util
+                from env_info import is_running_locally, get_resources_path
+                from messaging.ReconnectingMessageManager import ReconnectingMessageManager
+                from messaging.MessageHandler import MessageHandler
+                from flask import request
+                from flask import redirect
+
+                # init message handler
+                from db.repository import Repository
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+            try:
+
+                import main
+
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+
+        def test_routes(self):
+            try:
+                from routes import debug
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+
+            try:
+                from routes import layers
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+
+            try:
+                from routes import nodes
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+
+        def test_messaging(self):
+            try:
+                import network_constants as netconst
+                from security.token_manager import TokenManager
+                from db.entities import Layer
+
+                import json
+                import requests
+                from typing import Dict, List
+                from threading import Thread
+
+                import logging
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+            try:
+                from messaging import MessageHandler
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+
+        def test_db(self):
+            try:
+                import network_constants as netconst
+                from database.MongoRepositoryBase import MongoRepositoryBase
+                from db.entities import Layer
+
+                import pymongo
+                import json
+                from typing import List, Dict
+
+                # init logging to file
+                import logging
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+            try:
+                from db import repository
+                from db.entities import layer
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+
+
+    if __name__ == '__main__':
+        unittest.main()
+except Exception as e:
+                print ("Exception found:")
+                print (e)
\ No newline at end of file

src/data-hub/semantic-linking-microservice/app/tests/test_pipeline.py

@@ -11,6 +11,7 @@ from typing import List
 class DummyMongoRepo:
     '''Dummy class to be used for testing the MessageHandler'''
     last_trace = None
+    layernodes = []
     def insert_trace(self, trace):
         self.last_trace = trace
     
@@ -34,7 +35,9 @@ class DummyMongoRepo:
         ]
 
     def add_layer_nodes(self, nodes: List):
-        pass
+        self.layernodes.extend(nodes)
+        return
+        
 
 class Test_Pipeline(unittest.TestCase):
     handler = None
@@ -64,10 +67,11 @@ class Test_Pipeline(unittest.TestCase):
                 }
             }
         }
-
-    def testTraceProcessing(self):
+#original name testTraceProcessing
+    def test_handle_new_trace_newTraceMsg_correctlyInserted(self):
         msg = self._buildTraceMessage()
         self.handler.handle_new_trace(msg["content"])
+        self.assertEqual(len(self.handler._repository.layernodes),1)
     
 
 if __name__ == '__main__':

src/modules/messaging/MessageManager.py

 import pika
+from pika import spec
 from threading import Thread
 import network_constants as netconst
 
@@ -47,10 +48,18 @@ class MessageManager:
         if self._error_callback != None:
             self._error_callback("Connection closed")
 
+    def _message_confirmed_callback(self, frame):
+        if isinstance(frame.method, spec.Basic.Ack):
+            LOGGER.debug("message acknowledged")
+            return
+        else:
+            LOGGER.warning("Message was rejected by broker!")
+
     def _channel_created_callback(self, channel: pika.channel.Channel):
         # Assign both channels
         if self._send_channel == None:
             self._send_channel = channel
+            self._send_channel.confirm_delivery(ack_nack_callback=self._message_confirmed_callback)
         else:
             self._receive_channel = channel
             LOGGER.info("RabbitMQ connection established")
@@ -91,8 +100,11 @@ class MessageManager:
 
     def send_message(self, exchange_name, routing_key, message):
         '''Sends a message to the exchange'''
-        self._send_channel.basic_publish(exchange_name, routing_key, message)
-
+        try:
+            self._send_channel.basic_publish(exchange_name, routing_key, message, mandatory=True)
+        except pika.exceptions.UnroutableError:
+            raise pika.exceptions.UnroutableError
+            
     def disconnect(self):
         '''Stops listening for messages and closes the connection'''
         try:
@@ -100,4 +112,4 @@ class MessageManager:
             self._connection.close()
             LOGGER.info("RabbitMQ connection closed")
         except pika.exceptions.ConnectionWrongStateError:
-            LOGGER.warning("RabbitMQ connection already closed")
+            LOGGER.warning("RabbitMQ connection already closed")
\ No newline at end of file

src/modules/messaging/ReconnectingMessageManager.py

@@ -3,6 +3,8 @@ import time
 from messaging.MessageManager import MessageManager
 
 import logging
+import json
+import pika
 LOGGER = logging.getLogger(__name__)
 
 class ReconnectingMessageManager:
@@ -91,5 +93,13 @@ class ReconnectingMessageManager:
         '''Sends a message to the exchange'''
         try:
             self._message_manager.send_message(exchange_name, routing_key, message)
+        except pika.exceptions.UnroutableError:
+            message = json.loads(message)
+            if message["retries"] < 10:
+                message["retries"] += 1
+                message = json.dumps(message)
+                self.send_message(exchange_name, routing_key, message)
+            else:
+                LOGGER.error("Message went through too many send retries and was not delivered")
         except:
             LOGGER.error("Error while sending message")

src/participation-hub/business-logic-microservice/app/_add_use_case_scripts/car_sharing/tables/add_car.py

-from _add_use_case_scripts.car_sharing.tables.requestPost import postLayersToSwagger, postTableToSwagger
+from _add_use_case_scripts.requestPost import postLayersToSwagger, postTableToSwagger
 
 def add_table(use_case: str, table_name:str):
     '''

src/participation-hub/business-logic-microservice/app/_add_use_case_scripts/car_sharing/tables/add_hash.py

-from _add_use_case_scripts.car_sharing.tables.requestPost import postLayersToSwagger, postTableToSwagger
+from _add_use_case_scripts.requestPost import postLayersToSwagger, postTableToSwagger
 
 def add_table(use_case: str, table_name: str):
     '''

src/participation-hub/business-logic-microservice/app/_add_use_case_scripts/car_sharing/tables/add_offer.py

-from _add_use_case_scripts.car_sharing.tables.requestPost import postLayersToSwagger, postTableToSwagger
+from _add_use_case_scripts.requestPost import postLayersToSwagger, postTableToSwagger
 
 def add_table(use_case: str, table_name: str):
     '''

src/participation-hub/business-logic-microservice/app/_add_use_case_scripts/car_sharing/tables/add_publication.py

-from _add_use_case_scripts.car_sharing.tables.requestPost import postLayersToSwagger, postTableToSwagger
+from _add_use_case_scripts.requestPost  import postLayersToSwagger, postTableToSwagger
 
 def add_table(use_case: str, table_name: str):
     '''

src/participation-hub/business-logic-microservice/app/_add_use_case_scripts/car_sharing/tables/add_travel.py

-from _add_use_case_scripts.car_sharing.tables.requestPost import postLayersToSwagger, postTableToSwagger
+from _add_use_case_scripts.requestPost  import postLayersToSwagger, postTableToSwagger
 
 def add_table(use_case: str, table_name: str):
     '''

src/participation-hub/business-logic-microservice/app/_add_use_case_scripts/car_sharing/tables/add_user.py

 
-from _add_use_case_scripts.car_sharing.tables.requestPost import postLayersToSwagger, postTableToSwagger
-
+from _add_use_case_scripts.requestPost  import postLayersToSwagger, postTableToSwagger
 
 
 def add_table(use_case: str, table_name: str):

src/participation-hub/business-logic-microservice/app/_add_use_case_scripts/crowd_journalism/add_crowdjournalism_schema.py

+import sys
+import os
+from pathlib import Path
+from typing import Dict, Any
+import requests
+
+modules_paths = ['.', '../../../modules/']
+for modules_path in modules_paths:
+    if os.path.exists(modules_path):
+        sys.path.insert(1, modules_path)
+
+from _add_use_case_scripts.crowd_journalism.tables import add_video,add_tag,add_purchase,add_event,add_classification
+
+import network_constants as nc
+from security.token_manager import TokenManager
+
+def add_use_case(use_case: str):
+
+    jwt = TokenManager.getInstance().getToken()
+    url = f"https://articonf1.itec.aau.at:30420/api/use-cases"
+    response = requests.post(
+            url,
+            verify=False, 
+            proxies = { "http":None, "https":None },
+            headers = { "Authorization": f"Bearer {jwt}"},
+            json = {"name": use_case}
+        )
+    
+    print(url+": "+str(response.content))     
+
+if __name__ == "__main__":
+    use_case = "crowd-journalism"
+    
+    # disable ssl warnings :) 
+    requests.packages.urllib3.disable_warnings() 
+
+    add_use_case(use_case)
+    add_video.main(use_case)
+    add_tag.main(use_case)
+    add_classification.main(use_case)
+    add_event.main(use_case)
+    add_purchase.main(use_case)
+    
\ No newline at end of file

src/participation-hub/business-logic-microservice/app/_add_use_case_scripts/crowd_journalism/tables/add_classification.py

+from _add_use_case_scripts.requestPost  import postLayersToSwagger, postTableToSwagger
+def add_table(use_case: str, table_name: str):
+    '''
+    take the columns and add the mappings at the server
+    replace all "/"'s in the internal representation with a "_"
+    '''
+
+    columns = [
+        # "docType",
+        "objecttype",
+        "userid",
+        "videoid",
+        "informative",
+        "impact",
+        "trustiness",
+        "lastupdate"
+    ]
+    
+    columns = { c : c for c in columns }
+    columns["UniqueID"] = "userid+videoid"
+
+    table = {
+        "name": table_name,
+        "mappings": columns
+    }
+
+    postTableToSwagger(use_case,table)
+
+def add_layers(use_case:str, table_name: str):
+    layers = [
+        {   #Useless as all objects aree Classification?
+            "use_case": use_case,
+            "table": table_name,
+            "name": "Object_Type_Layer",
+            "properties": [
+                "UniqueID",
+                "objecttype",
+                "userid",
+                "videoid",
+                "informative",
+                "impact",
+                "trustiness",
+                "lastupdate"
+            ],
+            "cluster_properties": [
+                "objecttype"
+            ]
+        },
+        {
+            "use_case": use_case,
+            "table": table_name,
+            "name": "Informative_Layer",
+            "properties": [
+                "UniqueID",
+                "objecttype",
+                "userid",
+                "videoid",
+                "informative",
+                "impact",
+                "trustiness",
+                "lastupdate"
+            ],
+            "cluster_properties": [
+                "informative"
+            ]
+        },
+        {
+            "use_case": use_case,
+            "table": table_name,
+            "name": "Impact_Layer",
+            "properties": [
+                "UniqueID",
+                "objecttype",
+                "userid",
+                "videoid",
+                "informative",
+                "impact",
+                "trustiness",
+                "lastupdate"
+            ],
+            "cluster_properties": [
+                "impact"
+            ]
+        },
+        {
+            "use_case": use_case,
+            "table": table_name,
+            "name": "Trust_Layer",
+            "properties": [
+                "UniqueID",
+                "objecttype",
+                "userid",
+                "videoid",
+                "informative",
+                "impact",
+                "trustiness",
+                "lastupdate"
+            ],
+            "cluster_properties": [
+                "trustiness"
+            ]
+        }
+    ]
+
+    postLayersToSwagger(use_case,layers)    
+
+def main(use_case: str):
+    print("Classification")
+    table_name = "classification"
+    add_table(use_case,table_name)
+    add_layers(use_case,table_name)
\ No newline at end of file

src/participation-hub/business-logic-microservice/app/_add_use_case_scripts/crowd_journalism/tables/add_event.py

+from _add_use_case_scripts.requestPost  import postLayersToSwagger, postTableToSwagger
+def add_table(use_case: str, table_name: str):
+    '''
+    take the columns and add the mappings at the server
+    replace all "/"'s in the internal representation with a "_"
+    '''
+    #TODO: split eventEpicenter
+    #TODO: tags is an array, deal with arrays
+    columns = [
+        # "docType",
+        "objecttype",
+        "eventid",
+        #"tags",
+        "eventEpicenter", #TODO 
+        "range"
+    ]
+    
+    columns = { c : c for c in columns }
+    columns["UniqueID"] = "eventid"
+    columns["firstTag"] = "tags[0]"
+
+    table = {
+        "name": table_name,
+        "mappings": columns
+    }
+
+    postTableToSwagger(use_case,table)
+
+def add_layers(use_case:str, table_name: str):
+    layers = [
+        {   #Useless as all objects are of the same type???
+            "use_case": use_case,
+            "table": table_name,
+            "name": "Object_Type_Layer",
+            "properties": [
+                "UniqueID",
+                "objecttype",
+                "eventid",
+                "eventEpicenter",
+                "range",
+                "firstTag"
+            ],
+            "cluster_properties": [
+                "objecttype"
+            ]
+        },
+        {
+            "use_case": use_case,
+            "table": table_name,
+            "name": "Tag_Layer",
+            "properties": [
+                "UniqueID",
+                "objecttype",
+                "eventid",
+                "evenEpicenter",
+                "range",
+                "firstTag"
+            ],
+            "cluster_properties": [
+                "firstTag"
+            ]
+        },
+        {
+            "use_case": use_case,
+            "table": table_name,
+            "name": "Event_Epicenter_Layer",
+            "properties": [
+                "UniqueID",
+                "objecttype",
+                "eventid",
+                "evenEpicenter",
+                "range",
+                "firstTag"
+            ],
+            "cluster_properties": [
+                "eventEpicenter"
+            ]
+        }
+    ]
+
+    postLayersToSwagger(use_case,layers)    
+
+def main(use_case: str):
+    print("event")
+    table_name = "event"
+    add_table(use_case,table_name)
+    add_layers(use_case,table_name)
\ No newline at end of file

src/participation-hub/business-logic-microservice/app/_add_use_case_scripts/crowd_journalism/tables/add_purchase.py

+from _add_use_case_scripts.requestPost  import postLayersToSwagger, postTableToSwagger
+def add_table(use_case: str, table_name: str):
+    '''
+    take the columns and add the mappings at the server
+    replace all "/"'s in the internal representation with a "_"
+    '''
+
+    columns = [
+        # "docType",
+        "objecttype",
+        "timestamp",
+        "userid",
+        "videoid",
+        "price",
+        "ownerid"
+    ]
+    
+    columns = { c : c for c in columns }
+    columns["UniqueID"] = "userid+videoid+ownerid"
+
+    table = {
+        "name": table_name,
+        "mappings": columns
+    }
+
+    postTableToSwagger(use_case,table)
+
+def add_layers(use_case:str, table_name: str):
+    layers = [
+        {   #Useless as all objects aree Classification?
+            "use_case": use_case,
+            "table": table_name,
+            "name": "Object_Type_Layer",
+            "properties": [
+                "UniqueID",
+                "objecttype",
+                "timestamp",
+                "userid",
+                "videoid",
+                "price",
+                "ownerid"
+            ],
+            "cluster_properties": [
+                "objecttype"
+            ]
+        },
+        {
+            "use_case": use_case,
+            "table": table_name,
+            "name": "Price_Layer",
+            "properties": [
+                "UniqueID",
+                "objecttype",
+                "timestamp",
+                "userid",
+                "videoid",
+                "price",
+                "ownerid"
+            ],
+            "cluster_properties": [
+                "price"
+            ]
+        },
+        {
+            "use_case": use_case,
+            "table": table_name,
+            "name": "Owner_Layer",
+            "properties": [
+                "UniqueID",
+                "objecttype",
+                "timestamp",
+                "userid",
+                "videoid",
+                "price",
+                "ownerid"
+            ],
+            "cluster_properties": [
+                "ownerid"
+            ]
+        },
+        {
+            "use_case": use_case,
+            "table": table_name,
+            "name": "Buyer_Layer",
+            "properties": [
+                "UniqueID",
+                "objecttype",
+                "timestamp",
+                "userid",
+                "videoid",
+                "price",
+                "ownerid"
+            ],
+            "cluster_properties": [
+                "userid"
+            ]
+        }
+    ]
+
+    postLayersToSwagger(use_case,layers)    
+
+def main(use_case: str):
+    print("purchase")
+    table_name = "purchase"
+    add_table(use_case,table_name)
+    add_layers(use_case,table_name)
\ No newline at end of file

src/participation-hub/business-logic-microservice/app/_add_use_case_scripts/crowd_journalism/tables/add_tag.py

+from _add_use_case_scripts.requestPost  import postLayersToSwagger, postTableToSwagger
+def add_table(use_case: str, table_name: str):
+    '''
+    take the columns and add the mappings at the server
+    replace all "/"'s in the internal representation with a "_"
+    '''
+
+    columns = [
+        # "docType",
+        "objecttype",
+        "tag"
+    ]
+    
+    columns = { c : c for c in columns }
+    columns["UniqueID"] = "objecttype+tag"
+
+    table = {
+        "name": table_name,
+        "mappings": columns
+    }
+
+    postTableToSwagger(use_case,table)
+
+def add_layers(use_case:str, table_name: str):
+    layers = [
+        {   #Useless as all objects aree the same type??
+            "use_case": use_case,
+            "table": table_name,
+            "name": "Object_Type_Layer",
+            "properties": [
+                "UniqueID",
+                "objecttype",
+                "tag"
+            ],
+            "cluster_properties": [
+                "objecttype"
+            ]
+        },
+        {
+            "use_case": use_case,
+            "table": table_name,
+            "name": "Tag_Layer",
+            "properties": [
+                "UniqueID",
+                "objecttype",
+                "tag"
+            ],
+            "cluster_properties": [
+                "tag"
+            ]
+        }
+    ]
+
+    postLayersToSwagger(use_case,layers)    
+
+def main(use_case: str):
+    print("tag")
+    table_name = "tag"
+    add_table(use_case,table_name)
+    add_layers(use_case,table_name)
\ No newline at end of file

src/participation-hub/business-logic-microservice/app/_add_use_case_scripts/crowd_journalism/tables/add_video.py

+from _add_use_case_scripts.requestPost  import postLayersToSwagger, postTableToSwagger
+
+def add_table(use_case: str, table_name: str):
+    '''
+    take the columns and add the mappings at the server
+    replace all "/"'s in the internal representation with a "_"
+    '''
+
+    columns = [
+        # "docType",
+        "objecttype",
+        "videoid",
+        "duration",
+        "price",
+        "creator",
+        "creationTimestamp",
+        #"tags",
+        "geolocation",
+        "eventid",
+        "lastupdate",
+        "md5",
+        "informativeRating",
+        "impactRating",
+        "trustinessRating",
+        "ready",
+        "path",
+        "preview",
+        #"thumbnails" #not important?
+
+    ]
+    
+    columns = { c : c for c in columns }
+    columns["UniqueID"] = "videoid"
+    columns["encodedAudio"] = "codec//audio"
+    columns["encodedVideo"] = "codec//video"
+    columns["firstTag"] = "tags[0]"
+
+    table = {
+        "name": table_name,
+        "mappings": columns
+    }
+
+    postTableToSwagger(use_case,table)
+
+def add_layers(use_case:str, table_name: str):
+    layers = [
+        {   #Useless as all objects are Classification?
+            "use_case": use_case,
+            "table": table_name,
+            "name": "Object_Type_Layer",
+            "properties": [
+                "objecttype",
+                "videoid",
+                "duration",
+                "price",
+                "creator",
+                "creationTimestamp",
+                "lastupdate",
+                "firstTag"
+            ],
+            "cluster_properties": [
+                "objecttype"
+            ]
+        },
+        {
+            "use_case": use_case,
+            "table": table_name,
+            "name": "Price_Layer",
+            "properties": [
+                "UniqueID",
+                "objecttype",
+                "creationTimestamp",
+                "geolocation",
+                "videoid",
+                "price",
+                "informativeRating",
+                "impactRating",
+                "trustinessRating",
+                "firstTag"
+            ],
+            "cluster_properties": [
+                "price"
+            ]
+        },
+        {
+            "use_case": use_case,
+            "table": table_name,
+            "name": "Tag_Layer",
+            "properties": [
+                "UniqueID",
+                "objecttype",
+                "creationTimestamp",
+                "geolocation",
+                "videoid",
+                "price",
+                "informativeRating",
+                "impactRating",
+                "trustinessRating",
+                "firstTag"
+            ],
+            "cluster_properties": [
+                "firstTag"
+            ]
+        },
+        {
+            "use_case": use_case,
+            "table": table_name,
+            "name": "Informative_Layer",
+            "properties": [
+                "UniqueID",
+                "objecttype",
+                "creationTimestamp",
+                "geolocation",
+                "videoid",
+                "price",
+                "informativeRating",
+                "impactRating",
+                "trustinessRating",
+                "firstTag"
+            ],
+            "cluster_properties": [
+                "informativeRating"
+            ]
+        },
+        {
+            "use_case": use_case,
+            "table": table_name,
+            "name": "Impact_Layer",
+            "properties": [
+                "UniqueID",
+                "objecttype",
+                "creationTimestamp",
+                "geolocation",
+                "videoid",
+                "price",
+                "informativeRating",
+                "impactRating",
+                "trustinessRating",
+                "firstTag"
+            ],
+            "cluster_properties": [
+                "impactRating"
+            ]
+        },
+        {
+            "use_case": use_case,
+            "table": table_name,
+            "name": "Trust_Layer",
+            "properties": [
+                "UniqueID",
+                "objecttype",
+                "creationTimestamp",
+                "geolocation",
+                "videoid",
+                "price",
+                "informativeRating",
+                "impactRating",
+                "trustinessRating",
+                "firstTag"
+            ],
+            "cluster_properties": [
+                "trustinessRating"
+            ]
+        },
+        {
+            "use_case": use_case,
+            "table": table_name,
+            "name": "Location_Layer",
+            "properties": [
+                "UniqueID",
+                "objecttype",
+                "creationTimestamp",
+                "geolocation",
+                "videoid",
+                "price",
+                "informativeRating",
+                "impactRating",
+                "trustinessRating",
+                "firstTag"
+            ],
+            "cluster_properties": [
+                "geolocation"
+            ]
+        },
+        {
+            "use_case": use_case,
+            "table": table_name,
+            "name": "Video_Age_Layer",
+            "properties": [
+                "UniqueID",
+                "objecttype",
+                "creationTimestamp",
+                "geolocation",
+                "videoid",
+                "price",
+                "informativeRating",
+                "impactRating",
+                "trustinessRating",
+                "firstTag"
+            ],
+            "cluster_properties": [
+                "creationTimestamp"
+            ]
+        }
+    ]
+
+    postLayersToSwagger(use_case,layers)    
+
+def main(use_case: str):
+    print("video")
+    table_name = "video"
+    add_table(use_case,table_name)
+    add_layers(use_case,table_name)

src/participation-hub/business-logic-microservice/app/_add_use_case_scripts/reddit/add_smart_reddit_schema.py

+import sys
+import os
+from pathlib import Path
+from typing import Dict, Any
+import requests
+
+modules_path = '../../../modules/'
+if os.path.exists(modules_path):
+    sys.path.insert(1, modules_path)
+
+import network_constants as nc
+from security.token_manager import TokenManager
+
+import tables.add_reddit as reddit
+
+def add_use_case(use_case: str):
+    jwt = TokenManager.getInstance().getToken()
+    url = f"https://articonf1.itec.aau.at:30420/api/use-cases"
+    response = requests.post(
+            url,
+            verify=False, 
+            proxies = { "http":None, "https":None },
+            headers = { "Authorization": f"Bearer {jwt}"},
+            json = {"name": use_case}
+        )
+    
+    print(url+": "+str(response.status_code))
+        
+
+if __name__ == "__main__":
+    use_case = "reddit"
+    
+    # disable ssl warnings :) 
+    requests.packages.urllib3.disable_warnings() 
+
+    add_use_case(use_case)
+    reddit.main(use_case)
\ No newline at end of file

src/participation-hub/business-logic-microservice/app/_add_use_case_scripts/reddit/tables/add_reddit.py

+import network_constants as nc
+from security.token_manager import TokenManager
+
+import requests
+
+def add_table(use_case: str, table_name: str):
+    '''
+    take the columns and add the mappings at the server
+    replace all "/"'s in the internal representation with a "_"
+    '''
+
+    columns = {}
+
+
+    use_case = "reddit"
+    columns = [
+        # "doctype",
+        "id",
+        "user_id",
+        "title",
+        "content",
+        "permalink",
+        "upvotes",
+        "percentage_upvoted",
+        "n_comments",
+        "subreddit",
+        "created_at"
+    ]
+    
+    columns = { c : c for c in columns }
+    
+    columns["UniqueID"] = "user_id+subreddit+id"
+
+    table = {
+        "name": table_name,
+        "mappings": columns
+    }
+
+    url = f"https://articonf1.itec.aau.at:30420/api/use-cases/{use_case}/tables"
+    jwt = TokenManager.getInstance().getToken()
+    response = requests.post(
+            url,
+            verify=False, 
+            proxies = { "http":None, "https":None },
+            headers = { "Authorization": f"Bearer {jwt}"},
+            json = table
+        )
+
+    print(url+": "+str(response.status_code))
+
+    
+def add_layers(use_case:str, table_name: str):
+    layers = [
+        # {  #subreddit is string cannot cluster
+        #     "use_case": use_case,
+        #     "table": table_name,
+        #     "name": "Subreddit_Layer",
+        #     "properties": [
+        #         "UniqueID",
+        #         "subreddit",
+        #         "user_id",
+        #         "title",
+        #         "content",
+        #         "permalink",
+        #         "upvotes",
+        #         "percentage_upvoted",
+        #         "n_comments"
+        #     ],
+        #     "cluster_properties": [
+        #         "subreddit"
+        #     ]
+        # },
+        {
+            "use_case": use_case,
+            "table": table_name,
+            "name": "Upvotes_Layer", #TODO Probably do something like Total Votes? so we can get a popularity?
+            "properties": [
+                "UniqueID",
+                "subreddit",
+                "user_id",
+                "title",
+                "content",
+                "permalink",
+                "upvotes",
+                "percentage_upvoted",
+                "n_comments",
+                "created_at"
+            ],
+            "cluster_properties": [
+                "upvotes",
+                "percentage_upvoted"
+            ]
+        },
+        {
+            "use_case": use_case,
+            "table": table_name,
+            "name": "Percentage_Layer",
+            "properties": [
+                "UniqueID",
+                "subreddit",
+                "user_id",
+                "title",
+                "content",
+                "permalink",
+                "upvotes",
+                "percentage_upvoted",
+                "n_comments",
+                "created_at"
+            ],
+            "cluster_properties": [
+                "percentage_upvoted"
+            ]
+        },
+        {
+            "use_case": use_case,
+            "table": table_name,
+            "name": "Engagement_Layer",
+            "properties": [
+                "UniqueID",
+                "subreddit",
+                "user_id",
+                "title",
+                "content",
+                "permalink",
+                "upvotes",
+                "percentage_upvoted",
+                "n_comments",
+                "created_at"
+            ],
+            "cluster_properties": [
+                "n_comments"
+            ]
+        },
+        {
+            "use_case": use_case,
+            "table": table_name,
+            "name": "Time_Layer",
+            "properties": [
+                "UniqueID",
+                "subreddit",
+                "user_id",
+                "title",
+                "content",
+                "permalink",
+                "upvotes",
+                "percentage_upvoted",
+                "n_comments",
+                "created_at"
+            ],
+            "cluster_properties": [
+                "created_at"  
+            ]
+        }
+    ]
+
+    jwt = TokenManager.getInstance().getToken()
+
+    for layer in layers:
+        url = f"https://articonf1.itec.aau.at:30420/api/layers"
+
+        response = requests.post(
+            url,
+            verify=False, 
+            proxies = { "http":None, "https":None },
+            headers = { "Authorization": f"Bearer {jwt}"},
+            json = layer
+        )
+
+        print(url+": "+str(response.status_code))    
+
+def main(use_case: str):
+    print("reddit")
+    table_name = "reddit"
+    add_table(use_case,table_name)
+    add_layers(use_case,table_name)

src/participation-hub/business-logic-microservice/app/_add_use_case_scripts/car_sharing/tables/requestPost.py → src/participation-hub/business-logic-microservice/app/_add_use_case_scripts/requestPost.py

@@ -2,6 +2,7 @@
 import network_constants as nc
 from security.token_manager import TokenManager
 import requests
+from typing import List
 
 def postTableToSwagger(use_case:str, table:dict ):
 
@@ -20,7 +21,7 @@ def postTableToSwagger(use_case:str, table:dict ):
     print(url+": "+str(response.status_code)+" MSG:"+str(response.content))
 
 
-def postLayersToSwagger(use_case:str, layers:dict):
+def postLayersToSwagger(use_case:str, layers: List[dict]):
     
     jwt = TokenManager.getInstance().getToken()
 
@@ -35,4 +36,4 @@ def postLayersToSwagger(use_case:str, layers:dict):
             json = layer
         )
 
-    print(url+": "+str(response.status_code)+" MSG:"+str(response.content))
+        print(url+": "+str(response.status_code)+" MSG:"+str(response.content))
\ No newline at end of file

src/participation-hub/business-logic-microservice/app/_add_use_case_scripts/vialog/add_vialog_schema.py

+import sys
+import os
+from pathlib import Path
+from typing import Dict, Any
+import requests
+
+modules_paths = ['.', '../../../modules/']
+for modules_path in modules_paths:
+    if os.path.exists(modules_path):
+        sys.path.insert(1, modules_path)
+
+from _add_use_case_scripts.vialog.tables import add_user, add_video, add_change
+
+import network_constants as nc
+from security.token_manager import TokenManager
+
+def add_use_case(use_case: str):
+    #use_case = "vialog"
+
+    jwt = TokenManager.getInstance().getToken()
+    url = f"https://articonf1.itec.aau.at:30420/api/use-cases"
+    response = requests.post(
+            url,
+            verify=False, 
+            proxies = { "http":None, "https":None },
+            headers = { "Authorization": f"Bearer {jwt}"},
+            json = {"name": use_case}
+        )
+    
+    print(url+": "+str(response.content))     
+
+if __name__ == "__main__":
+    use_case = "vialog"
+    
+    # disable ssl warnings :) 
+    requests.packages.urllib3.disable_warnings() 
+
+    add_use_case(use_case)
+    add_user.main(use_case)
+    add_video.main(use_case)
+    add_change.main(use_case)
\ No newline at end of file

src/participation-hub/business-logic-microservice/app/_add_use_case_scripts/vialog/add_vialog_schema_new_enum.py

+import sys
+import os
+from pathlib import Path
+from typing import Dict, Any
+import requests
+
+modules_paths = ['.', '../../../modules/']
+for modules_path in modules_paths:
+    if os.path.exists(modules_path):
+        sys.path.insert(1, modules_path)
+
+from _add_use_case_scripts.vialog.tables import add_user, add_video, add_change
+
+import network_constants as nc
+from security.token_manager import TokenManager
+
+def add_use_case(use_case: str):
+    #use_case = "vialog"
+
+    jwt = TokenManager.getInstance().getToken()
+    url = f"https://articonf1.itec.aau.at:30420/api/use-cases"
+    response = requests.post(
+            url,
+            verify=False, 
+            proxies = { "http":None, "https":None },
+            headers = { "Authorization": f"Bearer {jwt}"},
+            json = {"name": use_case}
+        )
+    
+    print(url+": "+str(response.content))     
+
+if __name__ == "__main__":
+    use_case = "vialog-new-enum"
+    
+    # disable ssl warnings :) 
+    requests.packages.urllib3.disable_warnings() 
+
+    add_use_case(use_case)
+    add_video.main(use_case)
+    add_change.main(use_case)
\ No newline at end of file

src/participation-hub/business-logic-microservice/app/_add_use_case_scripts/vialog/tables/add_change.py

+from _add_use_case_scripts.requestPost  import postLayersToSwagger, postTableToSwagger
+def add_table(use_case: str, table_name: str):
+    '''
+    take the columns and add the mappings at the server
+    replace all "/"'s in the internal representation with a "_"
+    '''
+
+    columns = [
+        "changeType",
+        "changedValue",
+        "previousValue",
+        "newValue",
+    ]
+    
+    columns = { c : c for c in columns }
+    columns["UniqueID"] = "videoId+changeId"
+    columns["changeType"] = "enum(changeType)"
+    columns["changedValue"] = "enum(changedValue)"
+
+    table = {
+        "name": table_name,
+        "mappings": columns
+    }
+
+    postTableToSwagger(use_case,table)
+
+def add_layers(use_case:str, table_name: str):
+    layers = [
+        {
+            "use_case": use_case,
+            "table": table_name,
+            "name": "Changetype_Layer",
+            "properties": [
+                "changeType",
+            ],
+            "cluster_properties": [
+                "changeType",
+            ]
+        },
+        {
+            "use_case": use_case,
+            "table": table_name,
+            "name": "Changedvalue_Layer",
+            "properties": [
+                "changedValue",
+                "previousValue",
+                "newValue"
+            ],
+            "cluster_properties": [
+                "changedValue"
+            ]
+        }
+    ]
+
+    postLayersToSwagger(use_case,layers)    
+
+
+def main(use_case: str):
+    print("user")
+    table_name = "change"
+    add_table(use_case,table_name)
+    add_layers(use_case,table_name)
\ No newline at end of file

src/participation-hub/business-logic-microservice/app/_add_use_case_scripts/vialog/tables/add_user.py

+from _add_use_case_scripts.requestPost  import postLayersToSwagger, postTableToSwagger
+def add_table(use_case: str, table_name: str):
+    '''
+    take the columns and add the mappings at the server
+    replace all "/"'s in the internal representation with a "_"
+    '''
+
+    columns = [
+        # "docType",
+        "userId",
+        "rewardBalance"
+    ]
+    
+    columns = { c : c for c in columns }
+    columns["UniqueID"] = "userId"
+
+    table = {
+        "name": table_name,
+        "mappings": columns
+    }
+
+    postTableToSwagger(use_case,table)
+
+def add_layers(use_case:str, table_name: str):
+    layers = [
+        {
+            "use_case": use_case,
+            "table": table_name,
+            "name": "User_Layer",
+            "properties": [
+                "UniqueID",
+                "rewardBalance"
+            ],
+            "cluster_properties": [
+                "UniqueID",
+            ]
+        },
+        {
+            "use_case": use_case,
+            "table": table_name,
+            "name": "User_Balance_Layer",
+            "properties": [
+                "UniqueID",
+                "rewardBalance"
+            ],
+            "cluster_properties": [
+                "rewardBalance"
+            ]
+        }
+    ]
+
+    postLayersToSwagger(use_case,layers)    
+
+
+def main(use_case: str):
+    print("user")
+    table_name = "user"
+    add_table(use_case,table_name)
+    add_layers(use_case,table_name)
\ No newline at end of file

src/participation-hub/business-logic-microservice/app/_add_use_case_scripts/vialog/tables/add_video.py

+from _add_use_case_scripts.requestPost  import postLayersToSwagger, postTableToSwagger
+def add_table(use_case: str, table_name: str):
+    '''
+    take the columns and add the mappings at the server
+    replace all "/"'s in the internal representation with a "_"
+    '''
+
+    columns = [
+        'videoId',
+        'objType', 
+        'eventName',
+        'video_token', 
+        'replyTo', 
+        'created', 
+        'duration', 
+        'videoResolution', 
+        'label', 
+        'threadId', 
+        'position', 
+        'views', 
+        'moderatedBy', 
+        'moderationDate', 
+        'communityManagerNotes', 
+        'rewards', 
+        'video_state', 
+        'video_type'
+    ]
+    
+    columns = { c : c for c in columns }
+    columns["UniqueID"] = "videoId"
+    columns["moderatedBy"] = "enum(moderatedBy)"
+    columns["video_state"] = "enum(video_state)"
+    columns["video_type"] = "enum(video_type)"
+    columns["videoResolution"] = "enum(videoResolution)"
+
+    table = {
+        "name": table_name,
+        "mappings": columns
+    }
+
+    postTableToSwagger(use_case,table)
+
+def add_layers(use_case:str, table_name: str):
+    layers = [
+        {
+            "use_case": use_case,
+            "table": table_name,
+            "name": "Manager_Layer",
+            "properties": [
+                "UniqueID",
+                "moderationDate",
+                "moderatedBy",
+                "video_state",
+                "video_type"
+            ],
+            "cluster_properties": [
+                "moderatedBy",
+                "video_state"
+            ]
+        },
+        {
+            "use_case": use_case,
+            "table": table_name,
+            "name": "Video_Popularity_Layer",
+            "properties": [
+                "UniqueID",
+                "label",
+                "created",
+                "views",
+                "rewards",
+                "video_state",
+                "video_type"
+            ],
+            "cluster_properties": [
+                "views",
+                "video_type"
+            ]
+        },
+        {
+            "use_case": use_case,
+            "table": table_name,
+            "name": "Video_Age_Layer",
+            "properties": [
+                "UniqueID",
+                "label",
+                "created",
+                "views",
+                "rewards",
+                "video_state",
+                "video_type"
+            ],
+            "cluster_properties": [
+                "created"
+            ]
+        },
+        {
+            "use_case": use_case,
+            "table": table_name,
+            "name": "Rewards_Layer",
+            "properties": [
+                "UniqueID",
+                "label",
+                "created",
+                "views",
+                "rewards",
+                "video_state",
+                "video_type"
+            ],
+            "cluster_properties": [
+                "rewards"
+            ]
+        },
+        {
+            "use_case": use_case,
+            "table": table_name,
+            "name": "Video_Lenght_Layer",
+            "properties": [
+                "UniqueID",
+                "created",
+                "views",
+                "duration",
+                "video_state",
+                "video_type"
+            ],
+            "cluster_properties": [
+                "duration"
+            ]
+        },
+        {
+            "use_case": use_case,
+            "table": table_name,
+            "name": "Video_Resolution_Layer",
+            "properties": [
+                "UniqueID",
+                "created",
+                "views",
+                "videoResolution",
+                "video_state",
+                "video_type"
+            ],
+            "cluster_properties": [
+                "videoResolution"
+            ]
+        }
+    ]
+
+    postLayersToSwagger(use_case,layers)    
+
+def main(use_case: str):
+    print("Video")
+    table_name = "video"
+    add_table(use_case,table_name)
+    add_layers(use_case,table_name)
\ No newline at end of file

src/participation-hub/business-logic-microservice/app/requirements.txt

@@ -8,6 +8,7 @@ click==7.1.2
 clickclick==1.2.2
 colorama==0.4.3
 connexion==2.7.0
+coverage==5.3.1
 cryptography==3.1
 Flask==1.1.2
 idna==2.10

src/participation-hub/business-logic-microservice/app/test_coverage.py

+import unittest
+import sys
+for path in ['../', './']:
+    sys.path.insert(1, path)
+
+#####################################
+### Don't include for test report ###
+#####################################
+try:
+    class TestCoverage(unittest.TestCase):
+        def test_init_main(self):
+            try:
+                # python -m unittest discover
+                # add modules folder to interpreter path
+                import sys
+                import os
+                from pathlib import Path
+                from typing import Dict, Any
+
+                modules_path = '../../../modules/'
+                if os.path.exists(modules_path):
+                    sys.path.insert(1, modules_path)
+
+                # load swagger config
+                import connexion
+                from security import swagger_util
+                from env_info import is_running_locally, get_resources_path
+                from flask import request
+                from flask import redirect
+
+                app = connexion.App(__name__, specification_dir='configs/')
+
+                from db.entities.layer_adapter import LayerAdapter
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+
+            try:
+                import main
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+
+        def test_db_main(self):
+            try:
+                import network_constants as netconst
+                from database.MongoRepositoryBase import MongoRepositoryBase
+                from db.entities import layer_adapter
+                from db.entities import table
+                from db.entities import use_case
+
+                import pymongo
+                import json
+                from typing import List, Dict
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+
+            try:
+                from db import repository
+                from db import table_repository
+                from db import use_case_repository
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+            
+
+        def test_routes(self):
+            try:
+                from routes import layer
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+            try:
+                from routes import tables
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+            try:
+                from routes import use_case
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+
+        def test_services(self):
+            try:
+                from services import layer_adapter_service
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+
+        def test_use_case_scripts(self):
+            try:
+                import network_constants as nc
+                from security.token_manager import TokenManager
+                import requests
+                from typing import List
+                from _add_use_case_scripts import requestPost
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+            #######
+
+            #from _add_use_case_scripts.bank-app import add_bank_app_schema ##eror not importing? invalid folder name?
+            #from _add_use_case_scripts.bank-app.tables import add_bank_app_schema
+            try:
+                from _add_use_case_scripts.car_sharing import add_carsharing_schema
+                from _add_use_case_scripts.car_sharing.tables import add_car
+                from _add_use_case_scripts.car_sharing.tables import add_hash
+                from _add_use_case_scripts.car_sharing.tables import add_offer
+                from _add_use_case_scripts.car_sharing.tables import add_publication
+                from _add_use_case_scripts.car_sharing.tables import add_travel
+                from _add_use_case_scripts.car_sharing.tables import add_user
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+
+            try:
+                from _add_use_case_scripts.crowd_journalism import add_crowdjournalism_schema
+                from _add_use_case_scripts.crowd_journalism.tables import add_classification
+                from _add_use_case_scripts.crowd_journalism.tables import add_event
+                from _add_use_case_scripts.crowd_journalism.tables import add_purchase
+                from _add_use_case_scripts.crowd_journalism.tables import add_tag
+                from _add_use_case_scripts.crowd_journalism.tables import add_video
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+            
+            try:
+                from _add_use_case_scripts.debug import add_debug_schema
+                from _add_use_case_scripts.debug.tables import add_pizza_table
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+
+            #from _add_use_case_scripts.smart-energy import add_smart_energy_schema
+            #from _add_use_case_scripts.smart-energy.tables import add_smart_energy
+            try:
+                from _add_use_case_scripts.vialog import add_vialog_schema
+                from _add_use_case_scripts.vialog.tables import add_user 
+                from _add_use_case_scripts.vialog.tables import add_video
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+            
+    if __name__ == '__main__':
+        unittest.main()
+except Exception as e:
+    print ("Exception found:")
+    print (e)
\ No newline at end of file

src/participation-hub/business-logic-microservice/app/tests/test_layer_adapter.py

@@ -4,7 +4,7 @@ from db.entities.layer_adapter import LayerAdapter
 
 
 class Test_Layer_Adapter(unittest.TestCase):
-    def test_valid_adapter(self):
+    def test_LayerAdapter_newLayerAdapterObj_validInstantiation(self):
         adapter1 = LayerAdapter("layer1", "use_case", "table", ["a", "c"], ["a"])
         print(adapter1.to_serializable_dict)
 

src/rest-gateway/app/main.py

@@ -21,33 +21,37 @@ import connexion
 from security import swagger_util
 from env_info import is_running_locally, get_resources_path
 from flask import request
+from flask_cors import CORS
 from flask import redirect
 
-app = connexion.App(__name__, specification_dir='configs/')
 
-@app.app.before_request
-def before_request():
-    if request.url.startswith('http://'):
-        url = request.url.replace('http://', 'https://', 1)
-        code = 301
-        return redirect(url, code=code)
+# start app
+if __name__ == '__main__':
+    app = connexion.App(__name__, specification_dir='configs/')
+    CORS(app.app)
 
-@app.route('/', methods=['GET'])
-def api_root():
-    return redirect('/api/ui')
+    @app.app.before_request
+    def before_request():
+        if request.url.startswith('http://'):
+            url = request.url.replace('http://', 'https://', 1)
+            code = 301
+            return redirect(url, code=code)
 
-# SSL configuration
-certificate_path = get_resources_path()
-context = (os.path.normpath(f'{certificate_path}/articonf1.crt'), os.path.normpath(f'{certificate_path}/articonf1.key')) # certificate and key files
+    @app.route('/', methods=['GET'])
+    def api_root():
+        return redirect('/api/ui')
+
+    # SSL configuration
+    certificate_path = get_resources_path()
+    context = (os.path.normpath(f'{certificate_path}/articonf1.crt'), os.path.normpath(f'{certificate_path}/articonf1.key')) # certificate and key files
+
+    if is_running_locally():
+        print("Running locally...")
+        app.add_api(swagger_util.get_bundled_specs(Path("configs/swagger_local.yml")),
+                    resolver = connexion.RestyResolver("cms_rest_api"))
+    else:
+        app.add_api(swagger_util.get_bundled_specs(Path("configs/swagger.yml")),
+                    resolver = connexion.RestyResolver("cms_rest_api"))
 
-if is_running_locally():
-    print("Running locally...")
-    app.add_api(swagger_util.get_bundled_specs(Path("configs/swagger_local.yml")),
-                resolver = connexion.RestyResolver("cms_rest_api"))
-else:
-    app.add_api(swagger_util.get_bundled_specs(Path("configs/swagger.yml")),
-                resolver = connexion.RestyResolver("cms_rest_api"))
 
-# start app
-if __name__ == '__main__':
     app.run(host='0.0.0.0', port=5000, debug=True, use_reloader=False, ssl_context=context) # disable reloader so only subscribed once to rabbitmqö
\ No newline at end of file

src/rest-gateway/app/pull_video_data.py

+import requests
+import json
+
+
+videoListUrl = "https://dev758755.vialog.app/Videos/Meta/ListAll"
+videoUrl = "https://dev758755.vialog.app/stat/events?type=video&id="
+
+# token from Rest Gateway to authorize 
+JWT_TOKEN = 'eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJ1c2VybmFtZSI6InJlZ3VsYXJAaXRlYy5hYXUuYXQiLCJjcmVhdGVkX2F0IjoiMjAyMS0wNC0wNyAxMjo0OTo0MS43MTkzNjQiLCJ2YWxpZF91bnRpbCI6IjIwMjEtMDQtMDggMTI6NDk6NDEuNzE5MzY0In0.FN6qqBQeJSmXtS0-0dBiL-ojz6Ou7E5Tc9macrrhM4A'
+
+useCase = "vialog-new-enum"
+
+def send_transaction_to_rest_gateway(transaction: dict):
+    res = requests.post(
+        url = 'https://articonf1.itec.aau.at:30401/api/trace',
+        json = transaction,
+        headers = {"Authorization": f"Bearer {JWT_TOKEN}"},
+        verify = False # ignore ssl error
+    )
+
+    print(res)
+
+
+videosRequest = requests.get(videoListUrl)
+empty = set()
+html = set()
+
+if (videosRequest.status_code != 200):
+    print(f"Status: {videosRequest.status_code}")
+
+dataCount = 0
+for video in videosRequest.json():
+    dataCount += 1
+    id = video["videoId"]
+    videoRequest = requests.get(f"{videoUrl}{id}")
+    
+    if videoRequest.status_code != 200:
+        print(f"Status: {videoRequest.status_code}")
+    
+    if videoRequest.text != "" and not videoRequest.text.startswith("<!DOCTYPE html>"):
+        print(f"\n\n{videoUrl}{id}")
+        historyList = sorted(videoRequest.json()[0]["History"], key=lambda k: k['moderationDate'], reverse=True)
+        historyList.append(empty)
+
+        i = 0
+        changeList = []
+        for eventMap in historyList:
+            if historyList[i+1] == empty:
+                break
+
+            if i == 0:
+                lastState = eventMap
+                lastState["ApplicationType"] = useCase
+                lastState["docType"] = "video"
+                print(f"{lastState}\n\n\n")
+                send_transaction_to_rest_gateway(lastState)
+            
+            if eventMap["eventName"] == 'r1eabcbdc8f5378b2ba71a1b6fe2038b Created' or eventMap["eventName"] == 'Created':
+                change = {"changeType": "Created", "changedValue": "video_state", "previousValue": "", "newValue":"Created"}
+            elif eventMap["eventName"] == "Restore":
+                change = {"changeType": "Restore", "changedValue": "", "previousValue": "", "newValue":""}
+            elif eventMap["eventName"] == "PositionChange":
+                change = {"changeType": "PositionChange", "changedValue": "position", "previousValue": historyList[i+1]["position"], "newValue": eventMap["position"]}
+            elif eventMap["eventName"] == "Hide":
+                change = {"changeType": "Hide", "changedValue": "video_state", "previousValue": historyList[i+1]["video_state"], "newValue": eventMap["video_state"]}
+            elif eventMap["eventName"] == "Publish":
+                change = {"changeType": "Publish", "changedValue": "video_state", "previousValue": historyList[i+1]["video_state"], "newValue":eventMap["video_state"]}
+            elif eventMap["eventName"] == "CMNote":
+                change = {"changeType": "CMNote", "changedValue": "communityManagerNotes", "previousValue": historyList[i+1]["communityManagerNotes"], "newValue":eventMap["communityManagerNotes"]}
+            elif eventMap["eventName"] == "Move":
+                change = {"changeType": "Move", "changedValue": "position", "previousValue": historyList[i+1]["position"], "newValue":eventMap["position"]}
+            elif eventMap["eventName"] == "VideoType":
+                change = {"changeType": "VideoType", "changedValue": "video_type", "previousValue": historyList[i+1]["video_type"], "newValue":eventMap["video_type"]}
+            elif eventMap["eventName"] == "Delete":
+                change = {"changeType": "Delete", "changedValue": "", "previousValue": "", "newValue": ""}
+            elif eventMap["eventName"] == "Copy":
+                change = {"changeType": "Copy", "changedValue": "", "previousValue": "", "newValue":""}
+            elif eventMap["eventName"] == "CustomLabel":
+                change = {"changeType": "CustomLabel", "changedValue": "label", "previousValue": historyList[i+1]["label"], "newValue":eventMap["label"]}
+            
+            change["videoId"] = id
+            change["changeId"] = i
+            change["timestamp"] = eventMap["moderationDate"]
+            change["ApplicationType"] = useCase
+            change["docType"] = "change"
+            print(change)
+            send_transaction_to_rest_gateway(change)
+            i += 1
+    elif videoRequest.text == "":
+        empty.add(id)
+    else:
+        html.add(id)
+
+
+print(f"empty: {empty}\n\n")
+print(f"html page: {html}\n\n")
+print(f"history: {historyList}")
+print(dataCount)
+
+        
+
+
+
+    
+
+
+
+
+

src/rest-gateway/app/requirements.txt

@@ -12,6 +12,7 @@ connexion==2.7.0
 cryptography==2.9.2
 Deprecated==1.2.10
 Flask==1.1.2
+Flask-Cors==3.0.10
 idna==2.10
 importlib-metadata==1.7.0
 inflection==0.5.0

src/rest-gateway/app/routes/blockchain_trace.py

@@ -9,7 +9,7 @@ def receive():
     body = request.json
     
     if isBlockchainTraceValid(body):
-        message = {'type': 'blockchain-transaction', 'content': body}
+        message = {'type': 'blockchain-transaction', 'retries': 0, 'content': body}
         message_sender.send_message('inhub', 'trace-retrieval', json.dumps(message))
         return Response(status=201)
         

src/rest-gateway/app/test_coverage.py

+import unittest
+import sys
+for path in ['../', './']:
+    sys.path.insert(1, path)
+
+#####################################
+### Don't include for test report ###
+#####################################
+
+try:
+    class TestCoverage(unittest.TestCase):
+        def test_init_main(self):
+            try:
+                print("Entered test main")
+                # python -m unittest discover
+                # add modules folder to interpreter path
+                import sys
+                import os
+                import prance
+                from pathlib import Path
+                from typing import Dict, Any
+
+                modules_path = '../../modules/'
+                if os.path.exists(modules_path):
+                    sys.path.insert(1, modules_path)
+
+                # init logging to file
+                import logging
+                LOG_FORMAT = ('%(levelname) -5s %(asctime)s %(name)s:%(funcName) -35s %(lineno) -5d:  %(message)s')
+                logging.basicConfig(level=logging.INFO, format=LOG_FORMAT)
+                LOGGER = logging.getLogger(__name__)
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+            #################################
+            try:
+                import connexion
+                from security import swagger_util
+                from env_info import is_running_locally, get_resources_path
+                from flask import request
+                from flask import redirect          
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+            try:
+                import main #something in main is causing an infinite loop (probably an async task/ listener)
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+            print("Finished test main")
+
+        def test_database(self):
+            try:
+                print("Entered test db")
+                import network_constants as netconst
+                from database.MongoRepositoryBase import MongoRepositoryBase
+
+                import pymongo
+                import json
+                from db.entities.user import User
+                from typing import List
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+
+            try:
+                from db import repository
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+
+            print("Finished test db")
+
+        def test_services(self):
+            try:
+                print("Entered test services")
+                from functools import wraps
+                from flask import g, request, redirect, url_for
+
+                # global imports (dont't worry, red is normal)
+                from db.repository import Repository
+                from db.entities.user import User
+                from services.user_service import UserService
+                from env_info import get_resources_path
+
+                import jwt
+                from datetime import datetime, timedelta
+                from typing import Dict
+                import bcrypt
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+
+            try:
+                from services import login_wrapper
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+
+            try:
+                from services import token_service
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+            try:
+                from services import user_service
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+            print("Finished test services")
+
+        def test_routes(self):
+            try:
+                print("Entered test routes")
+                from flask import request, Response
+                from messaging.ReconnectingMessageManager import ReconnectingMessageManager 
+                import json
+                from flask import request
+                # global imports (dont't worry, red is normal)
+                from db.entities.user import User
+                from services.user_service import UserService
+                from services.login_wrapper import login_required
+                from services.token_service import TokenService
+                import bcrypt
+                import jwt
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+
+            try:
+                from routes import user
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+
+            #from routes import blockchain_trace  #message_sender in blockchain_trace is causing an infinite loop (probabily an async task//listener)
+            
+            try:
+                from routes import debug
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+            print("Finished test routes")
+
+        def test_add_users(self):
+            try:
+                print("Entered test users")
+                # add modules folder to interpreter path
+                import sys
+                import os
+                import json
+                import prance
+                from pathlib import Path
+                from typing import Dict, Any
+
+                modules_path = '../../modules/'
+                if os.path.exists(modules_path):
+                    sys.path.insert(1, modules_path)
+
+                from services.user_service import UserService
+                from env_info import is_running_locally, get_resources_path
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+
+            try:
+                import add_users
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+            print("Finished test users")
+
+
+
+    if __name__ == '__main__':
+        unittest.main()
+except Exception as e:
+    print ("Exception found:")
+    print (e)
\ No newline at end of file

src/transaction-hub-in/trace-retrieval-microservice/app/configs/routes.yml

@@ -62,6 +62,39 @@ paths:
       responses:
         '200':
           description: "Successful Request"
+  /use_cases/{use_case}/transactions-duplicated:
+    delete:
+      security:
+        - JwtRegular: []
+      operationId: "routes.transactions.delete_all_duplicated_for_use_case"
+      tags:
+        - "Transactions"
+      summary: "Deletes all duplicated Transactions in the given Use-Case"
+      description: "Deletes all duplicated Transactions in the given Use-Case"
+      parameters:
+        - in: path
+          name: "use_case"
+          required: true
+          type: "string"
+      responses:
+        '200':
+          description: "Successful Request"
+    get:
+      security:
+        - JwtRegular: []
+      operationId: "routes.transactions.all_duplicated_for_use_case"
+      tags:
+        - "Transactions"
+      summary: "Retrieves all duplicated Transactions in the given Use-Case"
+      description: "Retrieves all duplicated Transactions in the given Use-Case"
+      parameters:
+        - in: path
+          name: "use_case"
+          required: true
+          type: "string"
+      responses:
+        '200':
+          description: "Successful Request"
 
   /debug:
     post:

src/transaction-hub-in/trace-retrieval-microservice/app/database/repository.py

@@ -18,26 +18,35 @@ class Repository(MongoRepositoryBase):
 
         self._transaction_collection = 'transactions'
         self._failed_transaction_collection = 'transactions_failed'
+        self._duplicated_transaction_collection = "transactions_duplicated"
     
     def delete_all_transactions(self):
         collection = self._database[self._transaction_collection]
         collection.delete_many({})
 
     def add_transaction(self, transaction: Transaction):
-        reference = self.get_transaction_with_id(transaction.id())
+        '''
+        Adds a transaction to mongodb repository.
+
+        @throws
+        KeyError - Duplicate transaction ID
+        '''
+        use_case= transaction.use_case
+        table = transaction.table
+        reference = self.get_transaction_with_id(transaction.id(),use_case, table)
         if reference == None:
             super().insert_entry(self._transaction_collection, transaction.to_serializable_dict())
         else:
-            raise ValueError(f"ID {transaction['UniqueID']} already exists")
+            raise KeyError(f"ID {transaction['UniqueID']} already exists")
     
     def all_transactions_for_use_case(self, use_case: str) -> List[Transaction]:
         result = super().get_entries(self._transaction_collection, projection={'_id': False}, selection={"use_case": use_case})
         return [Transaction.from_serializable_dict(row) for row in list(result)]
     
-    def get_transaction_with_id(self, unique_id: str) -> Transaction:
-        result = list(super().get_entries(self._transaction_collection, projection={'_id': False}, selection={"UniqueID": unique_id}))
+    def get_transaction_with_id(self, unique_id: str, use_case:str, table:str ) -> Transaction:
+        result = list(super().get_entries(self._transaction_collection, projection={'_id': False}, selection={"id": unique_id,"use_case": use_case, "table":table}))
 
-        if len(result) == 1:
+        if len(result) >= 1:
             return Transaction.from_serializable_dict(result[0])
         
         return None
@@ -57,4 +66,16 @@ class Repository(MongoRepositoryBase):
     
     def delete_all_failed_transactions(self, use_case:str):
         collection = self._database[self._failed_transaction_collection]
+        collection.delete_many({"ApplicationType": use_case})
+
+    def add_duplicated_transaction(self, transaction: Transaction):
+        #transaction["timestamp"] = time.time()
+        super().insert_entry(self._duplicated_transaction_collection, transaction.to_serializable_dict())
+    
+    def all_duplicated_transactions_for_use_case(self, use_case: str) -> List[Dict]:
+        result = super().get_entries(self._duplicated_transaction_collection, projection={'_id': False}, selection={"use_case": use_case})
+        return [Transaction.from_serializable_dict(row) for row in list(result)]
+    
+    def delete_all_duplicated_transactions(self, use_case:str):
+        collection = self._database[self._duplicated_transaction_collection]
         collection.delete_many({"ApplicationType": use_case})
\ No newline at end of file

src/transaction-hub-in/trace-retrieval-microservice/app/main.py

@@ -27,7 +27,7 @@ from flask import request
 from flask import redirect
 
 # init message handler
-message_handler = MessageHandler(Repository(), ReconnectingMessageManager.getInstance(), RestFetcher())
+message_handler:MessageHandler = None
 def message_received_callback(channel, method, properties, body):
     message_handler.handle_generic(body)
 
@@ -59,6 +59,7 @@ else:
 
 # start app
 if __name__ == '__main__':
+    message_handler = MessageHandler(Repository(), ReconnectingMessageManager.getInstance(), RestFetcher())
     message_manager = ReconnectingMessageManager.getInstance()
     message_manager.start_consuming('inhub', 'direct', 'trace-retrieval', True, message_received_callback)
     

src/transaction-hub-in/trace-retrieval-microservice/app/messaging/MessageHandler.py

@@ -8,9 +8,11 @@ import json
 import hashlib
 import logging
 import requests
+requests.packages.urllib3.disable_warnings()
 
 from typing import Dict
 
+
 LOGGER = logging.getLogger(__name__)
 
 class MessageHandler:
@@ -30,7 +32,7 @@ class MessageHandler:
         self._rest_fetcher = rest_fetcher
 
     def handle_generic(self, body):
-        LOGGER.info(f"Received message: {body}")
+        
         result = None
 
         message = None
@@ -39,11 +41,13 @@ class MessageHandler:
         except (ValueError, TypeError):
             result = self.MSG_NOT_JSON
             LOGGER.warning(result)
+            LOGGER.info(f"Received message: {body}")
             return result
 
         if not 'type' in message:
             result = self.MSG_NO_TYPE
             LOGGER.warning(result)
+            LOGGER.info(f"Received message: {body}")
             return result
         
         if message['type'] == 'blockchain-transaction':
@@ -51,8 +55,11 @@ class MessageHandler:
             result = self.MSG_TRACE_PROCESSED
         else:
             result = self.MSG_NOT_PROCESSED
+            LOGGER.warning(result)
+            LOGGER.info(f"Received message: {body}")
+            
 
-        LOGGER.info(result)
+        #LOGGER.info(result) #too much spam
         return result
     
     def _resolve_path(self, data: Dict, path:str) -> Dict:
@@ -132,56 +139,74 @@ class MessageHandler:
         '''
 
         # check if there is a use-case in the message
-        if "ApplicationType" not in transaction_message.keys():
+        if "ApplicationType" not in transaction_message:
             LOGGER.error("Transaction has no ApplicationType, storing it under use-case 'unknown'.")
             transaction_message["ApplicationType"] = "unknown"
             self._mongo_repo.add_failed_transaction(transaction_message)
             return
 
-        # check if there is a doctype in the message
-        if "docType" not in transaction_message.keys():
-            LOGGER.error("Transaction has no docType, ignoring it.")
+        # check if there is a table in the message
+        if "docType" not in transaction_message:
+            LOGGER.error("Transaction has no docType, storing it under table 'unknown'.")
+            transaction_message["docType"] = "unknown"
+            self._mongo_repo.add_failed_transaction(transaction_message)
             return
 
-        use_case = transaction_message["ApplicationType"]
-        docType = transaction_message["docType"]
+        transaction_use_case = transaction_message["ApplicationType"]
+        transaction_table = transaction_message["docType"]
 
         try:
-            tables = self._rest_fetcher.fetch_schema_information(use_case)
+            tables = self._rest_fetcher.fetch_schema_information(transaction_use_case)
         except ValueError as e:
-            print(f"{e}")
+            LOGGER.error(f"{e}\nCould not fetch schema, storing it as a failed transaction..")
             self._mongo_repo.add_failed_transaction(transaction_message)
             return
 
         target_table = None
         # find correct table
         for table in tables:
-            if table["name"] == docType:
+            if table["name"] == transaction_table:
                 target_table = table
                 break
         
         # abort if table does not exist.
         if target_table == None:
-            LOGGER.error(f"There is no table '{docType}', ignoring the message.")
+            LOGGER.error(f"There is no table '{transaction_table}', storing it as a failed transaction.")
+            self._mongo_repo.add_failed_transaction(transaction_message)
             return
         
         mappings = table["mappings"]
         try:
             flattened = self._flatten_transaction(transaction_message, mappings)
         except KeyError as e:
-            LOGGER.error(f"Failed while flattening with KeyError: {str(e)}")
+            LOGGER.error(f"Failed while flattening with KeyError: {str(e)}, storing it as a failed transaction.")
             self._mongo_repo.add_failed_transaction(transaction_message)
             return
 
-        transaction = Transaction(use_case, target_table["name"], flattened)
+        transaction = Transaction(transaction_use_case, target_table["name"], flattened)
+
+        #check for duplicates
         try:
-            self._mongo_repo.add_transaction(transaction)
+            references = self._mongo_repo.get_transaction_with_id(transaction.id(),transaction_use_case,transaction_table)
+            if references != None:
+                LOGGER.info("Found duplicate, storing it as a duplicated transaction.")
+                self._mongo_repo.add_duplicated_transaction(transaction)
+                return 
         except ValueError as e:
-            LOGGER.error(f"{e}, ignoring node")
+            LOGGER.error(f"{e}, could not insert duplicated node.")
+            return
+
+        try:
+            self._mongo_repo.add_transaction(transaction)
+        except KeyError as e:
+            LOGGER.error(f"{e}, ignored {transaction_message}")
+            # self._mongo_repo.add_failed_transaction(transaction_message) 
+            return
+
 
         msg = {
             "type": "new-trace",
-            "content": transaction.to_serializable_dict(),
+            "content": transaction.to_serializable_dict()
         }
 
         msg_string = json.dumps(msg)

src/transaction-hub-in/trace-retrieval-microservice/app/messaging/dummy_rest_fetcher.py

@@ -2,13 +2,44 @@ from messaging.rest_fetcher import RestFetcher
 
 class DummyRestFetcher(RestFetcher):
     def fetch_schema_information(self, use_case: str):
-        return [
-            {
-                "name": "string",
-                "use_case": "string",
-                "mappings": {
-                    "UniqueID": "ResourceIds",
-                    "RIds": "ResourceIds"
-                }
-            }
-        ]
+        returnList = []
+        
+        if use_case == "string":
+            returnList =[
+                        {
+                            "name": "string",
+                            "use_case": "string",
+                            "mappings": {
+                                "UniqueID": "ResourceIds",
+                                "RIds": "ResourceIds"
+                            }
+                        },
+                        {
+                            "name": "string2",
+                            "use_case": "string",
+                            "mappings": {
+                                "UniqueID": "ResourceIds",
+                                "RIds": "ResourceIds"
+                            }
+                        }
+                    ]
+        else:
+            returnList = [
+                        {
+                            "name": "string",
+                            "use_case": "string2",
+                            "mappings": {
+                                "UniqueID": "ResourceIds",
+                                "RIds": "ResourceIds"
+                            }
+                        },
+                        {
+                            "name": "string2",
+                            "use_case": "string2",
+                            "mappings": {
+                                "UniqueID": "ResourceIds",
+                                "RIds": "ResourceIds"
+                            }
+                        }
+                    ]
+        return returnList
\ No newline at end of file

src/transaction-hub-in/trace-retrieval-microservice/app/messaging/rest_fetcher.py

@@ -18,7 +18,7 @@ class RestFetcher:
         )
 
         if response.status_code != 200:
-            raise ValueError("Error while retrieving schema information")
+            raise ValueError(f"Error while retrieving schema information.\tStatus-code:{response.status_code}")
 
         tables = json.loads(response.text)
         return tables

src/transaction-hub-in/trace-retrieval-microservice/app/requirements.txt

@@ -7,6 +7,7 @@ click==7.1.2
 clickclick==1.2.2
 colorama==0.4.3
 connexion==2.7.0
+coverage==5.3.1
 cryptography==3.1
 Deprecated==1.2.10
 Flask==1.1.2

src/transaction-hub-in/trace-retrieval-microservice/app/routes/transactions.py

@@ -20,4 +20,12 @@ def all_failed_for_use_case(use_case: str):
 
 def delete_all_failed_for_use_case(use_case: str):
     _repository.delete_all_failed_transactions(use_case)
+    return Response(status=200)
+
+def all_duplicated_for_use_case(use_case: str):
+    transactions = _repository.all_duplicated_transactions_for_use_case(use_case)
+    return [t.to_serializable_dict() for t in transactions]
+
+def delete_all_duplicated_for_use_case(use_case: str):
+    _repository.delete_all_duplicated_transactions(use_case)
     return Response(status=200)
\ No newline at end of file

src/transaction-hub-in/trace-retrieval-microservice/app/test_coverage.py

+import unittest
+import sys
+for path in ['../', './']:
+    sys.path.insert(1, path)
+
+#####################################
+### Don't include for test report ###
+#####################################
+
+try:
+    class TestCoverage(unittest.TestCase):
+        def test_init_main(self):
+            try:
+                # python -m unittest discover
+                # add modules folder to interpreter path
+                import sys
+                import os
+                import prance
+                from pathlib import Path
+
+                modules_path = '../../../modules/'
+                if os.path.exists(modules_path):
+                    sys.path.insert(1, modules_path)
+
+                # init logging to file
+                import logging
+                LOG_FORMAT = ('%(levelname) -5s %(asctime)s %(name)s:%(funcName) -35s %(lineno) -5d:  %(message)s')
+                logging.basicConfig(level=logging.INFO, format=LOG_FORMAT)
+                LOGGER = logging.getLogger(__name__)
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+
+            #############################
+            try:
+                import connexion
+                from security import swagger_util
+                from env_info import is_running_locally, get_resources_path 
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+
+            try:
+                from database.repository import Repository
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+
+            try:
+                from messaging.MessageHandler import MessageHandler
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+
+            try:
+                from messaging.ReconnectingMessageManager import ReconnectingMessageManager
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+
+            try:
+                from messaging.rest_fetcher import RestFetcher
+                from flask import request
+                from flask import redirect
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+            try:
+                import main
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+
+        def test_database(self):
+            # global imports (dont't worry, red is normal)
+            try:
+                from typing import List, Dict
+                import network_constants as netconst
+                from database.MongoRepositoryBase import MongoRepositoryBase
+                from database.entities.transaction import Transaction
+                
+                import pymongo
+                import json
+                import time
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+            try:
+                from database import repository
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+
+        def test_messaging(self):
+            try:
+                from security.token_manager import TokenManager
+                import network_constants
+                from database.entities.transaction import Transaction
+                from database.repository import Repository
+                from messaging.rest_fetcher import RestFetcher
+
+                import json
+                import hashlib
+                import logging
+                import requests
+
+                from typing import Dict
+                from typing import List
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+            try:
+                from messaging import MessageHandler
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+
+            try:
+                from messaging import rest_fetcher
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+
+        def test_routes(self):
+            #global imports
+            try:
+                from database.entities.transaction import Transaction
+                from database.repository import Repository
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+
+            try:
+                import json
+                from flask import Response, request
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+
+            try:
+                from routes import transactions
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+
+        
+
+    if __name__ == '__main__':
+        unittest.main()
+except Exception as e:
+    print ("Exception found:")
+    print (e)
\ No newline at end of file

src/transaction-hub-in/trace-retrieval-microservice/app/tests/test_MessageHandler.py

@@ -10,6 +10,7 @@ class DummyMongoRepo:
 
     def __init__(self):
         self.added_transactions = []
+        self.duplicated_transactions = []
 
     def insert_trace(self, trace):
         self.last_trace = trace
@@ -17,6 +18,22 @@ class DummyMongoRepo:
     def add_transaction(self, transaction):
         self.added_transactions.append(transaction)
 
+    def get_transaction_with_id(self, unique_id: str, use_case,table):
+        result = []
+        for trans in self.added_transactions:
+            transID = trans.id()
+            if transID == unique_id and trans.use_case == use_case and trans.table == table:
+                result.append(trans)
+        if len(result) > 0:
+            return result[0]
+        
+        return None
+
+    
+
+    def add_duplicated_transaction(self, transaction):
+        self.duplicated_transactions.append(transaction)
+
 from messaging.DummyMessageManager import DummyMessageManager as DummyMessageSender
 from messaging.dummy_rest_fetcher import DummyRestFetcher
 
@@ -53,6 +70,48 @@ class Test_MessageHandler(unittest.TestCase):
             }
         }
         return json.dumps(message_values)
+    def _get_valid_message2(self) -> str:
+        message_values = \
+        { 'type': 'blockchain-transaction',
+          'content': 
+            {
+            "ApplicationType": "string2",
+            "docType": "string",
+            "Metadata": {},
+            "ResourceIds": "string",
+            "ResourceMd5": "string",
+            "ResourceState": "string",
+            "Timestamp": "2019-08-27T14:00:48.587Z",
+            "TransactionFrom": "string",
+            "TransactionFromLatLng": "string",
+            "TransactionId": "string",
+            "TransactionTo": "string",
+            "TransactionToLatLng": "string",
+            "TransferredAsset": "string"
+            }
+        }
+        return json.dumps(message_values)
+    def _get_valid_message3(self) -> str:
+        message_values = \
+        { 'type': 'blockchain-transaction',
+          'content': 
+            {
+            "ApplicationType": "string",
+            "docType": "string2",
+            "Metadata": {},
+            "ResourceIds": "string",
+            "ResourceMd5": "string",
+            "ResourceState": "string",
+            "Timestamp": "2019-08-27T14:00:48.587Z",
+            "TransactionFrom": "string",
+            "TransactionFromLatLng": "string",
+            "TransactionId": "string",
+            "TransactionTo": "string",
+            "TransactionToLatLng": "string",
+            "TransferredAsset": "string"
+            }
+        }
+        return json.dumps(message_values)
 
     def test_handleGeneric_emptyMessage_NotJsonError(self):
         res = self.handler.handle_generic('')
@@ -111,5 +170,52 @@ class Test_MessageHandler(unittest.TestCase):
         self.assertEqual('semantic-linking', self.msg_sender.last_message['key'])
         self.assertEqual('new-trace', json.loads(self.msg_sender.last_message['msg'])["type"])
 
+    def test_handleBlockchainTransaction_duplicateTrace_oneTransAddedToDuplicateRepo(self):
+        msg = self._get_valid_message()
+        msg2 = self._get_valid_message()
+        msg = eval(msg)
+        msg2 = eval(msg2)
+        self.handler.handle_blockchain_transaction(msg['content'])
+        self.handler.handle_blockchain_transaction(msg2['content'])
+        self.assertEqual(len(self.repo.added_transactions),len(self.repo.duplicated_transactions))
+        self.assertEqual(len(self.repo.added_transactions),1)
+
+    def test_handleBlockchainTransaction_duplicateTraceDifferentTable_bothTransactionsAddedAsUnique(self):
+        msg = self._get_valid_message()
+        msg2 = self._get_valid_message2()
+        msg = eval(msg)
+        msg2 = eval(msg2)
+        self.handler.handle_blockchain_transaction(msg['content'])
+        self.handler.handle_blockchain_transaction(msg2['content'])
+        self.assertEqual(len(self.repo.added_transactions),2)
+
+    def test_handleBlockchainTransaction_duplicateTraceDifferentUseCase_bothTransactionsAddedAsUnique(self):
+        msg = self._get_valid_message()
+        msg2 = self._get_valid_message3()
+        msg = eval(msg)
+        msg2 = eval(msg2)
+        self.handler.handle_blockchain_transaction(msg['content'])
+        self.handler.handle_blockchain_transaction(msg2['content'])
+        self.assertEqual(len(self.repo.added_transactions),2)
+
+    def test_handleBlockchainTransaction_multipleTransactions3SameIdDiffUseCaseTable_3AddedUnique2Duplicate(self):
+        #print("Entered Test: 3Unique 2Dupli")
+        msg = self._get_valid_message()
+        msg2 = self._get_valid_message2()
+        msg3 = self._get_valid_message3()
+        msg4 = self._get_valid_message3()
+        msg5 = self._get_valid_message3()
+        msg = eval(msg)
+        msg2 = eval(msg2)
+        msg3 = eval(msg3)
+        msg4 = eval(msg4)
+        msg5 = eval(msg5)
+        self.handler.handle_blockchain_transaction(msg['content'])
+        self.handler.handle_blockchain_transaction(msg2['content'])
+        self.handler.handle_blockchain_transaction(msg3['content'])
+        self.handler.handle_blockchain_transaction(msg4['content'])
+        self.handler.handle_blockchain_transaction(msg5['content'])
+        self.assertEqual(len(self.repo.added_transactions),3)
+        self.assertEqual(len(self.repo.duplicated_transactions),2)
 if __name__ == '__main__':
     unittest.main()
\ No newline at end of file

tools/check-use-case-data/check_mog.py

+import requests
+requests.packages.urllib3.disable_warnings() 
+from icecream import ic
+
+
+def httpget(url):
+    token = 'eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJ1c2VybmFtZSI6InJlZ3VsYXJAaXRlYy5hYXUuYXQiLCJjcmVhdGVkX2F0IjoiMjAyMS0wNC0wNyAwODo0MDo0NS4yODEwOTYiLCJ2YWxpZF91bnRpbCI6IjIwMjEtMDQtMDggMDg6NDA6NDUuMjgxMDk2In0.oIDOEYy8bmIR3AHDRU-T0upYU0Wcz7V4FYzO5tSaSzk'
+    res = requests.get(url, 
+                    verify=False, 
+                    headers = { "Authorization": f"Bearer {token}"})
+    return res
+
+# list tables
+res = httpget(url = 'https://articonf1.itec.aau.at:30420/api/use-cases/crowd-journalism/tables')
+print("Tables: ", [entry['name'] for entry in res.json()])
+
+
+
+# count pushed data
+def count_data(json_res, table_identifier='table'):
+    tables = {}
+    for entry in json_res:
+        key = entry[table_identifier]
+        if key not in tables:
+            tables[key] = 0
+        tables[key] += 1
+    ic(tables)
+
+res = httpget(url = 'https://articonf1.itec.aau.at:30001/api/use_cases/crowd-journalism/transactions')
+count_data(res.json())
+
+res_f = httpget(url = 'https://articonf1.itec.aau.at:30001/api/use_cases/crowd-journalism/transactions-failed')
+count_data(res_f.json(), 'docType')
+
+res_d = httpget(url = 'https://articonf1.itec.aau.at:30001/api/use_cases/crowd-journalism/transactions-duplicated')
+count_data(res_d.json())
+
+# failed tags: the "tag" is missing, but is called name
+# failed purchases: duplicate keys generated from (userid, videoid, ownerid)
+# failed classifications: impact is missing
+
+
+# visualize content
+import matplotlib.pyplot as plt
+
+def visualize_video_coordinates():
+    geolocations = []
+    for entry in res.json():
+        if entry['table'] != 'video':
+            continue
+        loc_ = entry['properties']['geolocation'].split(',')
+        if loc_[0] == 'undefined' or loc_[1] == 'undefined':
+            continue
+        geolocations.append(loc_)
+
+    plt.scatter([float(coor[0]) for coor in geolocations], [float(coor[1]) for coor in geolocations])
+    plt.axis('off')
+    plt.show()
+# visualize_video_coordinates()
+
+
+def visualize_video_prices():
+    price = []
+    for entry in res.json():
+        if entry['table'] != 'video':
+            continue
+        price.append(entry['properties']['price'])
+
+    from collections import Counter
+    print(Counter(price))
+
+    plt.hist(price, bins=len(set(price)))
+    plt.show()
+# visualize_video_prices()
+
+
+def visualize_content_ratings():
+    impact = []
+    informative = []
+    trustiness = []
+    for entry in res.json():
+        if entry['table'] != 'classification':
+            continue
+        if entry['properties']['impact'] is not None:
+            impact.append(entry['properties']['impact'])
+        if entry['properties']['informative'] is not None:
+            informative.append(entry['properties']['informative'])
+        if entry['properties']['trustiness'] is not None:
+            trustiness.append(entry['properties']['trustiness'])
+
+    from collections import Counter
+    print(Counter(impact))
+    print(Counter(informative))
+    print(Counter(trustiness))
+
+    fig, (ax1, ax2, ax3) = plt.subplots(3)
+    ax1.hist(impact, bins=len(set(impact)))
+    ax1.set_title('impact')
+    ax2.hist(informative, bins=len(set(informative)))
+    ax2.set_title('informative')
+    ax3.hist(trustiness, bins=len(set(trustiness)))
+    ax3.set_title('trustiness')
+    plt.show()
+# visualize_content_ratings()
+
+
+# counting duplicate entries for 'purchase'
+working_purchase_ids = [(entry['properties']['userid'], entry['properties']['videoid'], entry['properties']['ownerid']) for entry in res.json() if entry['table'] == 'purchase']
+failed_purchase_ids = [(entry['userid'], entry['videoid'], entry['ownerid']) for entry in res_f.json() if entry['docType'] == 'purchase']
+
+ic(len(working_purchase_ids))
+ic(len(failed_purchase_ids))
+
+cnt = 0
+for failed_id in failed_purchase_ids:
+    if failed_id in working_purchase_ids:
+        cnt += 1
+
+ic(cnt)
\ No newline at end of file

tools/check-use-case-data/reupload_failed_mog.py

+import requests
+requests.packages.urllib3.disable_warnings() 
+from icecream import ic
+
+token = 'eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJ1c2VybmFtZSI6InJlZ3VsYXJAaXRlYy5hYXUuYXQiLCJjcmVhdGVkX2F0IjoiMjAyMS0wMy0yNCAxMDoxMzo1MS4wMjkwNDkiLCJ2YWxpZF91bnRpbCI6IjIwMjEtMDMtMjUgMTA6MTM6NTEuMDI5MDQ5In0.V6kYV5Lmb_tUIsF-6AKNB8_lIifmJP_Dm8gHhGa5w_o'
+
+def httpget(url):
+    res = requests.get(url, 
+                    verify=False, 
+                    headers = { "Authorization": f"Bearer {token}"})
+    return res
+
+res_f = httpget(url = 'https://articonf1.itec.aau.at:30001/api/use_cases/crowd-journalism/transactions-failed')
+
+failed_purchases = []
+for entry in res_f.json():
+    if entry['docType'] == 'purchase':
+        failed_purchases.append(entry)
+
+print(len(failed_purchases))
+
+
+# upload again
+def httppost_gateway(content_):
+    url = 'https://articonf1.itec.aau.at:30401/api/trace'
+    res = requests.post(url, 
+                    verify=False, 
+                    headers = { "Authorization": f"Bearer {token}"},
+                    json=content_)
+    return res
+
+for purchase in failed_purchases:
+    res = httppost_gateway(purchase)
+    print(res)
\ No newline at end of file

tools/reddit-upload/upload_data.py

+''' This script adds all data from BitYoga's csv to our pipeline.'''
+import csv
+import requests
+import sys
+import os
+import json
+
+import urllib3
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+# modules_path = '../../../modules/'
+# if os.path.exists(modules_path):
+#     sys.path.insert(1, modules_path)
+
+# import network_constants as nc
+# from security.token_manager import TokenManager
+
+
+def send_transaction_to_rest_gateway(transaction: dict):
+    # token from Rest Gateway to authorize 
+    JWT_TOKEN = 'eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJ1c2VybmFtZSI6InJlZ3VsYXJAaXRlYy5hYXUuYXQiLCJjcmVhdGVkX2F0IjoiMjAyMS0wMy0xNiAxMzoxNDoyMS42MDc1NjciLCJ2YWxpZF91bnRpbCI6IjIwMjEtMDMtMTcgMTM6MTQ6MjEuNjA3NTY3In0.ZGObriEDWYo1BgiYN3pQSosS7UuNrq10GSCSjmRHSAw'
+
+    res = requests.post(
+        url = 'https://articonf1.itec.aau.at:30401/api/trace',
+        json = transaction,
+        headers = {"Authorization": f"Bearer {JWT_TOKEN}"},
+        verify = False # ignore ssl error
+    )
+
+    if res.status_code >= 400:
+        raise Exception(f"Error while uploading: {str(res.content)}")
+
+    #print(res) Lots of spam
+
+# file to read the data from
+JSON_DATASET = r'reddit_dataset.json'
+if __name__ == '__main__':    
+    with open(JSON_DATASET, 'r') as json_file:
+        # reader = csv.reader(file)
+        json_data_list = json.load(json_file)
+        #titles = next(json_data)
+        
+        summ = 0          
+
+        for obj_dict in json_data_list:
+            transaction = {}
+            transaction['ApplicationType'] = 'reddit'
+            transaction['docType'] = 'reddit'
+
+            for key, value in obj_dict.items():
+                transaction[key] = value
+
+            send_transaction_to_rest_gateway(transaction)
+            summ+=1
+
+            if (summ % 1000 == 0 ):
+                print ("Uploaded " + str(summ) + " transactions.")
+            if summ >= 1:
+                break
+        print ("TOTAL Uploaded " + str(summ) + " transactions.")
+      
\ No newline at end of file