Merge branch 'develop' into feature/dashboard

.gitignore

@@ -23,3 +23,5 @@ src/dashboard/.dart_tool/
 src/dashboard/build/61d113a1f91ed254ee3636485549491e/
 
 src/dashboard/build/
+
+reports/

bin/deploy.py

@@ -61,7 +61,7 @@ def delete_deployment(name) -> int:
 
 if __name__ == '__main__':
     deployment_file_paths = []
-    for p, _, f in os.walk('./'):
+    for p, _, f in os.walk('./src/'):
         for file in f:
             if 'deployment.yml' == file:
                 deployment_file_paths.append(os.path.normpath(p))

bin/testing.py

@@ -2,7 +2,8 @@ import os
 import sys
 import importlib.util
 import pathlib
-
+import shutil
+import re
 '''
 This script searches for all 'tests/' directories and executes all tests 
 by cd'ing into the dir and executing unittest discover.
@@ -12,29 +13,107 @@ Use command line argument '-w' to run on windows.
 
 PY = sys.argv[2] if (len(sys.argv) > 1 and sys.argv[1] == '-py') else 'python3.7' # use -py to use your own python command
 ROOT = pathlib.Path(__file__).parent.parent.absolute()
+REPORTS = ROOT / 'reports'
 TESTS_FOLDER_NAME = os.path.normpath("/tests")
 
+print("Creating VENV")
+os.system(f"{PY} -m venv venv")
+
+PY = f"~/smart/venv/bin/{PY}"
+
 print("\nSearching for tests at the path: "+ str(ROOT))
 
 count = 0
 resultCodeList = []
+microservice_coverage_paths_set = set()
 for (dirname, dirs, files) in os.walk(ROOT):
 
     #I assume all the tests are placed in a folder named "tests"
     if (TESTS_FOLDER_NAME in str(dirname)) \
         and 'src' in str(dirname) \
         and not(f"{TESTS_FOLDER_NAME}{os.path.normpath('/')}" in str(dirname)) \
-        and not("venv" in str(dirname)):
+        and not("venv" in str(dirname)) \
+        and not("Lib" in str(dirname)):
         try:
             print(f"Executing tests in {dirname}")
             os.chdir(os.path.normpath(dirname))
             # TODO do this during docker image setup
             exit_val = os.system(f"{PY} -m pip install -r ../requirements.txt") # install pip dependencies
-            exit_val = os.system(f"{PY} -m unittest discover") # execute the tests
+            #resultCodeList.append(exit_val)
+            #exit_val = os.system(f"{PY} -m unittest discover") # execute the tests
+            exit_val = os.system(f"{PY} -m coverage run --append --omit=*/site-packages*,*/dist-packages* -m unittest discover") #TEST CODE COVERAGE
+            microservice_coverage_paths_set.add(os.path.normpath(dirname))
+            
             resultCodeList.append(exit_val) #once per folder  i.e if 3 tests are in a folder and crash, there will be just one exit val
         except Exception as e:
             print(e)
             continue
+        try:
+            cur_dir = pathlib.Path(os.path.normpath(dirname)).parent.absolute()
+            filename_regular_expresion = re.compile('(test_.*)|(TEST_.*)')
+            
+
+            for filename in os.listdir(cur_dir):
+                if filename_regular_expresion.match(filename): 
+                    #gets here only if there is a test file which matches the regular expression in the app folder,
+                    
+                    #cur_dir = os.path(dirname).parent()
+                    os.chdir(cur_dir)
+                    print(f"Executing coverage test in {cur_dir}")
+                    exit_val = os.system(f"{PY} -m coverage run --append --omit=*/site-packages* -m unittest discover")
+                    microservice_coverage_paths_set.add(os.path.normpath(cur_dir))
+
+        except Exception as e:
+            print(e)
+            continue
+        
+
+
+#CHANGE FOLDER TO REPORTS, in order to combine the coverage
+try:
+    if not os.path.exists(REPORTS):
+        os.makedirs(REPORTS) 
+except:
+    pass
+
+
+try: 
+    os.chdir(REPORTS)
+    target = REPORTS
+    target = os.path.normpath( str(target) + f'/.coverage' )
+    os.remove(target) #Try to Remove old coverage file, if exists
+except Exception as e:
+    pass
+
+print("Combinging coverages")
+counter = 0
+for path in microservice_coverage_paths_set:
+    try:
+        path += '/.coverage'
+        original = os.path.normpath( path )
+        target = REPORTS
+        target = os.path.normpath( str(target) + f'/.coverage.{counter}' )
+        counter += 1
+
+        shutil.copyfile(original,target) #copy new generated coverage files
+        os.remove(original)
+    except Exception as e:
+        print(e)
+        continue
+try:
+    coverage_xml_path = os.path.normpath( str(REPORTS) + '/coverage.xml')
+    os.remove(coverage_xml_path)
+
+    #coverage_html_path = os.path.normpath( str(REPORTS) + '/htmlcov' )
+    #os.rmdir(coverage_html_path)
+except Exception as e:
+    print(e)
+
+
+print("Generating Combined report")
+os.system(f"{PY} -m coverage combine")
+os.system(f"{PY} -m coverage xml")
+os.system(f"{PY} -m coverage html") #if you want to generate the html as well
 
 firstError = -1
 i = 0
@@ -48,4 +127,5 @@ while i < len(resultCodeList):
 if(firstError<0): #no errors found
     sys.exit(0)
 else:
-    sys.exit(1) #return code>0
\ No newline at end of file
+    sys.exit(1) #return code>0
+

documentation/external_access.md

@@ -24,6 +24,7 @@ This token is used for authentication as _regular user_ on all microservices cur
 ```
 {
   "ApplicationType": "use-case identifier as string",
+  "docType": "use-case-table identifier as string",
   "key": "value",
   ...
 }
@@ -91,4 +92,4 @@ Returns the computed similarity. Two clusters belonging to the SAME layer will b
 ## Connected Cluster
 Intermediary data-structure used only by the function which computes the similarity. Clusters are connected only to other clusters belonging to a DIFFERENT layer.
 
-```GET https://articonf1.itec.aau.at:30103/api/use_cases/{use_case}/tables{table}/connectedClusters``` returns all connected clusters for the given use-case and table.
+```GET https://articonf1.itec.aau.at:30103/api/use_cases/{use_case}/tables/{table}/connectedClusters``` returns all connected clusters for the given use-case and table.

src/data-hub/pareto-trust-microservice/Dockerfile

 FROM python:3
-LABEL maintainer="Alexander Lercher"
-
-ENV http_proxy http://proxy.uni-klu.ac.at:3128/
-ENV https_proxy http://proxy.uni-klu.ac.at:3128/
+LABEL maintainer="Manuel Herold"
 
 RUN apt-get update
 RUN pip install flask

src/data-hub/proactive-community-detection-microservice/Dockerfile

 FROM python:3
 LABEL maintainer="Alexander Lercher"
 
-ENV http_proxy http://proxy.uni-klu.ac.at:3128/
-ENV https_proxy http://proxy.uni-klu.ac.at:3128/
-
 RUN apt-get update
 RUN pip install flask
 RUN pip install connexion[swagger-ui]

src/data-hub/reputation-calculation-microservice/Dockerfile

 FROM python:3
-LABEL maintainer="Alexander Lercher"
-
-ENV http_proxy http://proxy.uni-klu.ac.at:3128/
-ENV https_proxy http://proxy.uni-klu.ac.at:3128/
+LABEL maintainer="Manuel Herold"
 
 RUN apt-get update
 RUN pip install flask

src/data-hub/role-stage-discovery-microservice/Dockerfile

 FROM python:3
 LABEL maintainer="Alexander Lercher"
 
-ENV http_proxy http://proxy.uni-klu.ac.at:3128/
-ENV https_proxy http://proxy.uni-klu.ac.at:3128/
-
 RUN apt-get update
 
 EXPOSE 5000

src/data-hub/role-stage-discovery-microservice/app/main.py

@@ -18,9 +18,11 @@ from pathlib import Path
 from env_info import is_running_locally, get_resources_path
 from flask import request
 from flask import redirect
+from flask_cors import CORS
 
 # load swagger config 
 app = connexion.App(__name__, specification_dir='configs/')
+CORS(app.app)
 
 @app.app.before_request
 def before_request():

src/data-hub/role-stage-discovery-microservice/app/requirements.txt

@@ -8,11 +8,13 @@ Click==7.0
 clickclick==1.2.2
 colorama==0.4.3
 connexion==2.6.0
+coverage==5.3.1
 cryptography==3.1
 cycler==0.10.0
 decorator==4.4.1
 Deprecated==1.2.7
 Flask==1.1.1
+Flask-Cors==3.0.10
 idna==2.8
 importlib-metadata==1.5.0
 inflection==0.3.1

src/data-hub/role-stage-discovery-microservice/app/test_coverage.py

+import unittest
+import sys
+for path in ['../', './']:
+    sys.path.insert(1, path)
+
+#####################################
+### Don't include for test report ###
+#####################################
+try: 
+    class TestCoverage(unittest.TestCase):
+        def test_init_main(self):
+            try:
+                # python -m unittest discover
+                from db.entities import Cluster
+
+
+                from datetime import date, datetime
+                import json
+
+                # add modules folder to interpreter path
+                import sys
+                import os
+                modules_path = '../../../modules/'
+                if os.path.exists(modules_path):
+                    sys.path.insert(1, modules_path)
+
+                ### init logging ###
+                import logging
+                LOG_FORMAT = ('%(levelname) -5s %(asctime)s %(name)s:%(funcName) -35s %(lineno) -5d:  %(message)s')
+                logging.basicConfig(level=logging.INFO, format=LOG_FORMAT)
+                LOGGER = logging.getLogger(__name__)
+
+                #############################
+                import connexion
+                from security import swagger_util
+                from pathlib import Path
+                from env_info import is_running_locally, get_resources_path
+                from flask import request
+                from flask import redirect
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+
+            try:
+                import main  #error when importing main, ModuleNotFoundError: No module named 'security'
+            #exec(open('main.py').read())
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+
+        def test_init_run_clustering(self):
+            try:
+                import sys
+                import os
+                modules_path = '../../../modules/'
+                if os.path.exists(modules_path):
+                    sys.path.insert(1, modules_path)
+
+                import json
+                from db.entities import Layer, Cluster
+                from typing import List, Dict, Tuple, Any
+                from db.repository import Repository
+                from processing.clustering import Clusterer, ClusterResult
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+            
+            try:
+                import run_clustering
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+
+        def test_init_run_node(self):
+            try:
+                import sys
+                import os
+                modules_path = '../../../modules/'
+                if os.path.exists(modules_path):
+                    sys.path.insert(1, modules_path)
+
+                import json
+                import urllib3
+                urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+            
+            try:
+                import processing.fetching.fetching as f
+                import run_node_fetching
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+
+        def test_init_run_similarity(self):
+            try:
+                import processing.similarityFiles.similarityMain as SimilarityCalc
+                from db.repository import Repository
+
+                import run_similarity_calc
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+
+        def test_init_run_time(self):
+            try:
+                import sys
+                import os
+                modules_path = '../../../modules/'
+                if os.path.exists(modules_path):
+                    sys.path.insert(1, modules_path)
+
+                import json
+                from datetime import datetime, date
+                from db.repository import Repository
+                from db.entities import ClusterSet, Cluster, Layer, TimeSlice
+                from typing import Tuple, Dict, Any, List
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+
+            try:
+                import run_time_slicing
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+
+
+    if __name__ == '__main__':
+        unittest.main()
+except Exception as e:
+    print ("Exception found:")
+    print (e)
\ No newline at end of file

src/data-hub/role-stage-discovery-microservice/app/tests/test_cluster.py

+
 import unittest
 import sys
 for path in ['../', './']:
@@ -19,6 +20,5 @@ class TestCluster(unittest.TestCase):
         self.assertEqual(1, c.cluster_label)
         self.assertEqual([1, 2, 3], c.nodes)
 
-
 if __name__ == '__main__':
     unittest.main()

src/data-hub/role-stage-discovery-microservice/app/tests/test_cluster_result.py

@@ -11,7 +11,9 @@ class TestClusterResult(unittest.TestCase):
     converter:ClusterResultConverter = None
 
     def setUp(self):
+    
         self.converter = ClusterResultConverter()
+    
 
     def test_result_undefined_feature(self):
         cluster_groups = self._get_some_cluster_groups_1d()
@@ -29,11 +31,13 @@ class TestClusterResult(unittest.TestCase):
             cluster_groups=cluster_groups,
             features=['v']
         )
+        
 
         self.assert_correct_cluster_result_len(cluster_groups, cluster_res)
         self.assert_correct_cluster_result_labels(['-1.0 -- 1.0','10.0 -- 11.0','2.0 -- 2.0'], cluster_res)
 
     def test_result_2d_features(self):
+
         cluster_groups = self._get_some_cluster_groups_2d()
         cluster_res = self.converter.convert_to_cluster_results(
             cluster_groups=cluster_groups,
@@ -42,6 +46,7 @@ class TestClusterResult(unittest.TestCase):
 
         self.assert_correct_cluster_result_len(cluster_groups, cluster_res)
         self.assert_correct_cluster_result_labels([str((0.0,0.0)), str((10.5,10.5)), str((2.0,2.0)), str((3.0,6.0))], cluster_res)
+
     
 
     #region Custom Assertions
@@ -52,6 +57,7 @@ class TestClusterResult(unittest.TestCase):
             self.assertEqual(len(expected[i]), len(actual[i].nodes))
             self.assertEqual(expected[i], actual[i].nodes)
 
+
     def assert_correct_cluster_result_labels(self, expected: List[str], actual: Dict[Any, ClusterResult]):
         self.assertEqual(len(expected), len(actual))
         for i in range(len(expected)):

src/data-hub/role-stage-discovery-microservice/app/tests/test_clusterer.py

+
+
 import unittest
 import sys
 for path in ['../', './']:
     sys.path.insert(1, path)
 
+
 # python -m unittest discover
 from processing.clustering import Clusterer, ClusterResult
 import numpy as np
@@ -187,8 +190,9 @@ class TestClusterer(unittest.TestCase):
                 self.fail(f"Cluster key ({k}, {type(k)}) not in result.")
             
             self.assertListEqual(expected[k], actual[k].nodes)
-   
+
     #endregion helper methods
 
 if __name__ == '__main__':
     unittest.main()
+

src/data-hub/role-stage-discovery-microservice/app/tests/test_similarity.py

@@ -23,7 +23,7 @@ import json
 class TestSimilarity(unittest.TestCase):
     '''Tests the similarity calculation which works without object orientation.'''
 
-    def test_integration_similarityCalculation(self):
+    def test_integration_calculateSimilarity_ClustersDict_CorrectValue(self):
         '''
         Only for testing, can be deleted at any time.\n
         Served as a testing example to make sure the computations are correct

src/data-hub/semantic-linking-microservice/Dockerfile

 FROM python:3
 LABEL maintainer="Alexander Lercher"
 
-ENV http_proxy http://proxy.uni-klu.ac.at:3128/
-ENV https_proxy http://proxy.uni-klu.ac.at:3128/
-
 RUN apt-get update
 
 EXPOSE 5000

src/data-hub/semantic-linking-microservice/app/main.py

@@ -20,6 +20,7 @@ from env_info import is_running_locally, get_resources_path
 from messaging.ReconnectingMessageManager import ReconnectingMessageManager
 from messaging.MessageHandler import MessageHandler
 from flask import request
+from flask_cors import CORS
 from flask import redirect
 
 # init message handler
@@ -30,6 +31,7 @@ def message_received_callback(channel, method, properties, body):
 
 # load swagger config
 app = connexion.App(__name__, specification_dir='configs/')
+CORS(app.app)
 
 @app.app.before_request
 def before_request():

src/data-hub/semantic-linking-microservice/app/requirements.txt

@@ -7,8 +7,10 @@ click==7.1.2
 clickclick==1.2.2
 colorama==0.4.3
 connexion==2.7.0
+coverage==5.3.1
 cryptography==3.1
 Flask==1.1.2
+Flask-Cors==3.0.10
 idna==2.9
 importlib-metadata==1.6.1
 inflection==0.5.0

src/data-hub/semantic-linking-microservice/app/test_coverage.py

+import unittest
+import sys
+for path in ['../', './']:
+    sys.path.insert(1, path)
+
+#####################################
+### Don't include for test report ###
+#####################################
+try:
+    class TestCoverage(unittest.TestCase):
+        def test_init_main(self):
+            try:
+                # add modules folder to interpreter path
+                import sys
+                import os
+                import prance
+                from pathlib import Path
+                modules_path = '../../../modules/'
+                if os.path.exists(modules_path):
+                    sys.path.insert(1, modules_path)
+
+                # init logging to file
+                import logging
+                LOG_FORMAT = ('%(levelname) -5s %(asctime)s %(name)s:%(funcName) -35s %(lineno) -5d:  %(message)s')
+                logging.basicConfig(level=logging.INFO, format=LOG_FORMAT)
+                LOGGER = logging.getLogger(__name__)
+
+                #################################
+                import connexion
+                from security import swagger_util
+                from env_info import is_running_locally, get_resources_path
+                from messaging.ReconnectingMessageManager import ReconnectingMessageManager
+                from messaging.MessageHandler import MessageHandler
+                from flask import request
+                from flask import redirect
+
+                # init message handler
+                from db.repository import Repository
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+            try:
+
+                import main
+
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+
+        def test_routes(self):
+            try:
+                from routes import debug
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+
+            try:
+                from routes import layers
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+
+            try:
+                from routes import nodes
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+
+        def test_messaging(self):
+            try:
+                import network_constants as netconst
+                from security.token_manager import TokenManager
+                from db.entities import Layer
+
+                import json
+                import requests
+                from typing import Dict, List
+                from threading import Thread
+
+                import logging
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+            try:
+                from messaging import MessageHandler
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+
+        def test_db(self):
+            try:
+                import network_constants as netconst
+                from database.MongoRepositoryBase import MongoRepositoryBase
+                from db.entities import Layer
+
+                import pymongo
+                import json
+                from typing import List, Dict
+
+                # init logging to file
+                import logging
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+            try:
+                from db import repository
+                from db.entities import layer
+            except Exception as e:
+                print ("Exception found:")
+                print (e)
+
+
+    if __name__ == '__main__':
+        unittest.main()
+except Exception as e:
+                print ("Exception found:")
+                print (e)
\ No newline at end of file

src/data-hub/semantic-linking-microservice/app/tests/test_pipeline.py

@@ -67,8 +67,8 @@ class Test_Pipeline(unittest.TestCase):
                 }
             }
         }
-
-    def testTraceProcessing(self):
+#original name testTraceProcessing
+    def test_handle_new_trace_newTraceMsg_correctlyInserted(self):
         msg = self._buildTraceMessage()
         self.handler.handle_new_trace(msg["content"])
         self.assertEqual(len(self.handler._repository.layernodes),1)

src/modules/network_constants.py

@@ -84,4 +84,31 @@ else:
     BUSINESS_LOGIC_REST_PORT = 30420
     BUSINESS_LOGIC_DB_PORT = 30421
 
+## Federated Learning
+if server:
+    FEDERATED_LEARNING_HOSTNAME = 'federated-learning'
+    #FEDERATED_LEARNING_DB_HOSTNAME = f'{EDERATED_LEARNING_HOSTNAME}-db'
+    FEDERATED_LEARNING_REST_PORT = 80
+    #FEDERATED_LEARNING_DB_PORT = 27017
+else:
+    FEDERATED_LEARNING_HOSTNAME = 'articonf1.itec.aau.at'
+    #FEDERATED_LEARNING_DB_HOSTNAME = 'articonf1.itec.aau.at'
+    FEDERATED_LEARNING_REST_PORT = 30422
+    #FEDERATED_LEARNING_DB_PORT = 30423
+
 #endregion Participation Hub
+
+#region Federated Training
+## Federated Training
+if server:
+    FEDERATED_TRAINING_HOSTNAME = 'gpu3.itec.aau.at'
+    #FEDERATED_TRAINING_DB_HOSTNAME = f'{FEDERATED_TRAINING_HOSTNAME}-db'
+    FEDERATED_TRAINING_REST_PORT = 30424
+    #FEDERATED_TRAINING_DB_PORT = 27017