Merge branch 'develop' into 'staging'

Develop

See merge request !34

bin/testing.py

@@ -4,7 +4,6 @@ import importlib.util
 import pathlib
 import shutil
 import re
-
 '''
 This script searches for all 'tests/' directories and executes all tests 
 by cd'ing into the dir and executing unittest discover.
@@ -12,16 +11,21 @@ It additionally installs all dependencies from a '../requirements.txt' via pip.
 Use command line argument '-w' to run on windows.
 '''
 
-PY = sys.argv[2] if (len(sys.argv) > 1 and sys.argv[1] == '-py') else 'python' # use -py to use your own python command
+PY = sys.argv[2] if (len(sys.argv) > 1 and sys.argv[1] == '-py') else 'python3.7' # use -py to use your own python command
 ROOT = pathlib.Path(__file__).parent.parent.absolute()
 REPORTS = ROOT / 'reports'
 TESTS_FOLDER_NAME = os.path.normpath("/tests")
 
+print("Creating VENV")
+os.system(f"{PY} -m venv venv")
+
+PY = f"~/smart/venv/bin/{PY}"
+
 print("\nSearching for tests at the path: "+ str(ROOT))
 
 count = 0
 resultCodeList = []
-coverage_paths_set = set()
+microservice_coverage_paths_set = set()
 for (dirname, dirs, files) in os.walk(ROOT):
 
     #I assume all the tests are placed in a folder named "tests"
@@ -37,8 +41,8 @@ for (dirname, dirs, files) in os.walk(ROOT):
             exit_val = os.system(f"{PY} -m pip install -r ../requirements.txt") # install pip dependencies
             #resultCodeList.append(exit_val)
             #exit_val = os.system(f"{PY} -m unittest discover") # execute the tests
-            exit_val = os.system(f"python3.7 -m coverage run --append --omit=*/site-packages*,*/dist-packages* -m unittest discover") #TEST CODE COVERAGE
-            coverage_paths_set.add(os.path.normpath(dirname))
+            exit_val = os.system(f"{PY} -m coverage run --append --omit=*/site-packages*,*/dist-packages* -m unittest discover") #TEST CODE COVERAGE
+            microservice_coverage_paths_set.add(os.path.normpath(dirname))
             
             resultCodeList.append(exit_val) #once per folder  i.e if 3 tests are in a folder and crash, there will be just one exit val
         except Exception as e:
@@ -56,8 +60,8 @@ for (dirname, dirs, files) in os.walk(ROOT):
                     #cur_dir = os.path(dirname).parent()
                     os.chdir(cur_dir)
                     print(f"Executing coverage test in {cur_dir}")
-                    exit_val = os.system(f"python3.7 -m coverage run --append --omit=*/site-packages* -m unittest discover")
-                    coverage_paths_set.add(os.path.normpath(cur_dir))
+                    exit_val = os.system(f"{PY} -m coverage run --append --omit=*/site-packages* -m unittest discover")
+                    microservice_coverage_paths_set.add(os.path.normpath(cur_dir))
 
         except Exception as e:
             print(e)
@@ -83,7 +87,7 @@ except Exception as e:
 
 print("Combinging coverages")
 counter = 0
-for path in coverage_paths_set:
+for path in microservice_coverage_paths_set:
     try:
         path += '/.coverage'
         original = os.path.normpath( path )
@@ -96,11 +100,20 @@ for path in coverage_paths_set:
     except Exception as e:
         print(e)
         continue
+try:
+    coverage_xml_path = os.path.normpath( str(REPORTS) + '/coverage.xml')
+    os.remove(coverage_xml_path)
+
+    #coverage_html_path = os.path.normpath( str(REPORTS) + '/htmlcov' )
+    #os.rmdir(coverage_html_path)
+except Exception as e:
+    print(e)
+
 
 print("Generating Combined report")
-os.system("python3.7 -m coverage combine")
-os.system("python3.7 -m coverage xml")
-os.system("python3.7 -m coverage html") #if you want to generate the html as well
+os.system(f"{PY} -m coverage combine")
+os.system(f"{PY} -m coverage xml")
+os.system(f"{PY} -m coverage html") #if you want to generate the html as well
 
 firstError = -1
 i = 0

src/data-hub/role-stage-discovery-microservice/app/test_coverage.py

@@ -3,7 +3,9 @@ import sys
 for path in ['../', './']:
     sys.path.insert(1, path)
 
-
+#####################################
+### Don't include for test report ###
+#####################################
 try: 
     class TestCoverage(unittest.TestCase):
         def test_init_main(self):

src/data-hub/role-stage-discovery-microservice/app/tests/test_cluster.py

-try:
-    import unittest
-    import sys
-    for path in ['../', './']:
+
+import unittest
+import sys
+for path in ['../', './']:
     sys.path.insert(1, path)
 
-    # python -m unittest discover
-    from db.entities import Cluster
+# python -m unittest discover
+from db.entities import Cluster
 
-    from datetime import date, datetime
-    import json
+from datetime import date, datetime
+import json
 
 
-    class TestCluster(unittest.TestCase):
+class TestCluster(unittest.TestCase):
     def test_init_Cluster(self):
-            try:
         c = Cluster('debug', 'debug-table1', 'layer1', 1, [1, 2, 3])
 
         self.assertEqual('debug', c.use_case)
         self.assertEqual('debug-table1', c.use_case_table)
         self.assertEqual(1, c.cluster_label)
         self.assertEqual([1, 2, 3], c.nodes)
-            except Exception as e:
-                print(e)
 
-    if __name__ == '__main__':
+if __name__ == '__main__':
     unittest.main()
-except Exception as e:
-    print ("Exception found:")
-    print (e)

src/data-hub/role-stage-discovery-microservice/app/tests/test_cluster_result.py

-try:
-    import unittest
-    import sys
-    for path in ['../', './']:
+import unittest
+import sys
+for path in ['../', './']:
     sys.path.insert(1, path)
 
-    # python -m unittest discover
-    from processing.clustering import ClusterResultConverter, ClusterResult
-    from typing import List, Dict, Any
+# python -m unittest discover
+from processing.clustering import ClusterResultConverter, ClusterResult
+from typing import List, Dict, Any
 
-    class TestClusterResult(unittest.TestCase):
+class TestClusterResult(unittest.TestCase):
     converter:ClusterResultConverter = None
 
     def setUp(self):
-            try:
+    
         self.converter = ClusterResultConverter()
-            except Exception as e:
-                print (e)
+    
 
     def test_result_undefined_feature(self):
-            try:
         cluster_groups = self._get_some_cluster_groups_1d()
         cluster_res = self.converter.convert_to_cluster_results(
             cluster_groups=cluster_groups,
@@ -27,11 +24,8 @@ try:
 
         self.assert_correct_cluster_result_len(cluster_groups, cluster_res)
         self.assert_correct_cluster_result_labels(['n.a.','n.a.','n.a.'], cluster_res)
-            except Exception as e:
-                print (e)
 
     def test_result_1d_feature(self):
-            try:
         cluster_groups = self._get_some_cluster_groups_1d()
         cluster_res = self.converter.convert_to_cluster_results(
             cluster_groups=cluster_groups,
@@ -42,12 +36,8 @@ try:
         self.assert_correct_cluster_result_len(cluster_groups, cluster_res)
         self.assert_correct_cluster_result_labels(['-1.0 -- 1.0','10.0 -- 11.0','2.0 -- 2.0'], cluster_res)
 
-            except Exception as e:
-                print (e)
-
     def test_result_2d_features(self):
 
-            try:
         cluster_groups = self._get_some_cluster_groups_2d()
         cluster_res = self.converter.convert_to_cluster_results(
             cluster_groups=cluster_groups,
@@ -57,29 +47,21 @@ try:
         self.assert_correct_cluster_result_len(cluster_groups, cluster_res)
         self.assert_correct_cluster_result_labels([str((0.0,0.0)), str((10.5,10.5)), str((2.0,2.0)), str((3.0,6.0))], cluster_res)
 
-            except Exception as e:
-                print (e)
     
 
     #region Custom Assertions
 
     def assert_correct_cluster_result_len(self, expected: 'original dict of lists', actual: Dict[Any, ClusterResult]):
-            try:
         self.assertEqual(len(expected), len(actual))
         for i in range(len(expected)):
             self.assertEqual(len(expected[i]), len(actual[i].nodes))
             self.assertEqual(expected[i], actual[i].nodes)
 
-            except Exception as e:
-                print (e)
 
     def assert_correct_cluster_result_labels(self, expected: List[str], actual: Dict[Any, ClusterResult]):
-            try:
         self.assertEqual(len(expected), len(actual))
         for i in range(len(expected)):
             self.assertEqual(expected[i], actual[i].label)
-            except Exception as e:
-                print (e)
 
     #endregion Custom Assertions
 
@@ -100,8 +82,5 @@ try:
         }
     #endregion helper methods
 
-    if __name__ == '__main__':
+if __name__ == '__main__':
     unittest.main()
\ No newline at end of file
-except Exception as e:
-    print ("Exception found:")
-    print (e)
\ No newline at end of file

src/data-hub/role-stage-discovery-microservice/app/tests/test_clusterer.py

 
-try:
-    import unittest
-    import sys
-    for path in ['../', './']:
+
+import unittest
+import sys
+for path in ['../', './']:
     sys.path.insert(1, path)
 
 
-    # python -m unittest discover
-    from processing.clustering import Clusterer, ClusterResult
-    import numpy as np
-    from typing import List, Dict, Any
+# python -m unittest discover
+from processing.clustering import Clusterer, ClusterResult
+import numpy as np
+from typing import List, Dict, Any
 
-    class TestClusterer(unittest.TestCase):
+class TestClusterer(unittest.TestCase):
     clusterer:Clusterer = None
 
     def setUp(self):
@@ -193,8 +193,6 @@ try:
 
     #endregion helper methods
 
-    if __name__ == '__main__':
+if __name__ == '__main__':
     unittest.main()
-except Exception as e:
-    print ("Exception found:")
-    print (e)
+

src/data-hub/role-stage-discovery-microservice/app/tests/test_similarity.py

-try:
-    import unittest
+import unittest
 
-    import sys
-    for path in ['../', './', '../../../modules/', '../../../../modules']:
+import sys
+for path in ['../', './', '../../../modules/', '../../../../modules']:
     sys.path.insert(1, path)
 
-    from db.entities.connected_node import NodeC
-    from db.entities.connected_cluster import ClusterC
-    from db.entities.connected_layer import LayerC
+from db.entities.connected_node import NodeC
+from db.entities.connected_cluster import ClusterC
+from db.entities.connected_layer import LayerC
 
-    import math
-    import datetime
-    from typing import Dict
-    from typing import Dict
-    from processing.similarityFiles.dataInput import *
-    from processing.similarityFiles.calculateWeights import *
-    from processing.similarityFiles.calculateSimilarity import *
-    from processing.similarityFiles.miscFunctions import *
+import math
+import datetime
+from typing import Dict
+from typing import Dict
+from processing.similarityFiles.dataInput import *
+from processing.similarityFiles.calculateWeights import *
+from processing.similarityFiles.calculateSimilarity import *
+from processing.similarityFiles.miscFunctions import *
 
-    import json
+import json
 
 
-    class TestSimilarity(unittest.TestCase):
+class TestSimilarity(unittest.TestCase):
     '''Tests the similarity calculation which works without object orientation.'''
 
-        def test_integration_similarityCalculation(self):
+    def test_integration_calculateSimilarity_ClustersDict_CorrectValue(self):
         '''
         Only for testing, can be deleted at any time.\n
         Served as a testing example to make sure the computations are correct
@@ -68,7 +67,7 @@ try:
 
 
 
-    def getTestLocationLayerData():
+def getTestLocationLayerData():
     inputLayerLocation = [
                             {
                                 "cluster_label": 1,
@@ -199,7 +198,7 @@ try:
                         ]
     return inputLayerLocation
 
-    def getTestTimeLayerData():
+def getTestTimeLayerData():
     inputLayerTime = [
                         {
                             "cluster_label": 4,
@@ -298,7 +297,7 @@ try:
                     ]
     return inputLayerTime
 
-    def getTestPriceLayerData():
+def getTestPriceLayerData():
     inputLayerPrice = [
                         {
                             "cluster_label": 6,
@@ -398,8 +397,5 @@ try:
 
 
 
-    if __name__ == '__main__':
+if __name__ == '__main__':
     unittest.main()
\ No newline at end of file
-except Exception as e:
-    print ("Exception found:")
-    print (e)
\ No newline at end of file

src/data-hub/semantic-linking-microservice/app/test_coverage.py

@@ -3,7 +3,9 @@ import sys
 for path in ['../', './']:
     sys.path.insert(1, path)
 
-
+#####################################
+### Don't include for test report ###
+#####################################
 try:
     class TestCoverage(unittest.TestCase):
         def test_init_main(self):

src/data-hub/semantic-linking-microservice/app/tests/test_pipeline.py

@@ -67,8 +67,8 @@ class Test_Pipeline(unittest.TestCase):
                 }
             }
         }
-
-    def testTraceProcessing(self):
+#original name testTraceProcessing
+    def test_handle_new_trace_newTraceMsg_correctlyInserted(self):
         msg = self._buildTraceMessage()
         self.handler.handle_new_trace(msg["content"])
         self.assertEqual(len(self.handler._repository.layernodes),1)

src/participation-hub/business-logic-microservice/app/test_coverage.py

@@ -3,7 +3,9 @@ import sys
 for path in ['../', './']:
     sys.path.insert(1, path)
 
-
+#####################################
+### Don't include for test report ###
+#####################################
 try:
     class TestCoverage(unittest.TestCase):
         def test_init_main(self):

src/participation-hub/business-logic-microservice/app/tests/test_layer_adapter.py

@@ -4,7 +4,7 @@ from db.entities.layer_adapter import LayerAdapter
 
 
 class Test_Layer_Adapter(unittest.TestCase):
-    def test_valid_adapter(self):
+    def test_LayerAdapter_newLayerAdapterObj_validInstantiation(self):
         adapter1 = LayerAdapter("layer1", "use_case", "table", ["a", "c"], ["a"])
         print(adapter1.to_serializable_dict)
 

src/rest-gateway/app/test_coverage.py

@@ -2,6 +2,11 @@ import unittest
 import sys
 for path in ['../', './']:
     sys.path.insert(1, path)
+
+#####################################
+### Don't include for test report ###
+#####################################
+
 try:
     class TestCoverage(unittest.TestCase):
         def test_init_main(self):

src/transaction-hub-in/trace-retrieval-microservice/app/test_coverage.py

@@ -3,6 +3,10 @@ import sys
 for path in ['../', './']:
     sys.path.insert(1, path)
 
+#####################################
+### Don't include for test report ###
+#####################################
+
 try:
     class TestCoverage(unittest.TestCase):
         def test_init_main(self):

src/transaction-hub-in/trace-retrieval-microservice/app/tests/test_MessageHandler.py

@@ -170,7 +170,7 @@ class Test_MessageHandler(unittest.TestCase):
         self.assertEqual('semantic-linking', self.msg_sender.last_message['key'])
         self.assertEqual('new-trace', json.loads(self.msg_sender.last_message['msg'])["type"])
 
-    def test_handleblockchain_duplicateTrace(self):
+    def test_handleBlockchainTransaction_duplicateTrace_oneTransAddedToDuplicateRepo(self):
         msg = self._get_valid_message()
         msg2 = self._get_valid_message()
         msg = eval(msg)
@@ -179,7 +179,7 @@ class Test_MessageHandler(unittest.TestCase):
         self.handler.handle_blockchain_transaction(msg2['content'])
         self.assertEqual(len(self.repo.added_transactions),len(self.repo.duplicated_transactions))
 
-    def test_handleblockchain_duplicateTraceDifferentTable(self):
+    def test_handleBlockchainTransaction_duplicateTraceDifferentTable_bothTransactionsAddedAsUnique(self):
         msg = self._get_valid_message()
         msg2 = self._get_valid_message2()
         msg = eval(msg)
@@ -188,7 +188,7 @@ class Test_MessageHandler(unittest.TestCase):
         self.handler.handle_blockchain_transaction(msg2['content'])
         self.assertEqual(len(self.repo.added_transactions),2)
 
-    def test_handleblockchain_duplicateTraceDifferentUseCase(self):
+    def test_handleBlockchainTransaction_duplicateTraceDifferentUseCase_bothTransactionsAddedAsUnique(self):
         msg = self._get_valid_message()
         msg2 = self._get_valid_message3()
         msg = eval(msg)