Merge branch 'develop' into 'staging'

Develop

See merge request !34

bin/testing.py

@@ -4,7 +4,6 @@ import importlib.util
 import pathlib
 import shutil
 import re
-
 '''
 This script searches for all 'tests/' directories and executes all tests 
 by cd'ing into the dir and executing unittest discover.
@@ -12,16 +11,21 @@ It additionally installs all dependencies from a '../requirements.txt' via pip.
 Use command line argument '-w' to run on windows.
 '''
 
-PY = sys.argv[2] if (len(sys.argv) > 1 and sys.argv[1] == '-py') else 'python' # use -py to use your own python command
+PY = sys.argv[2] if (len(sys.argv) > 1 and sys.argv[1] == '-py') else 'python3.7' # use -py to use your own python command
 ROOT = pathlib.Path(__file__).parent.parent.absolute()
 REPORTS = ROOT / 'reports'
 TESTS_FOLDER_NAME = os.path.normpath("/tests")
 
+print("Creating VENV")
+os.system(f"{PY} -m venv venv")
+
+PY = f"~/smart/venv/bin/{PY}"
+
 print("\nSearching for tests at the path: "+ str(ROOT))
 
 count = 0
 resultCodeList = []
-coverage_paths_set = set()
+microservice_coverage_paths_set = set()
 for (dirname, dirs, files) in os.walk(ROOT):
 
     #I assume all the tests are placed in a folder named "tests"
@@ -37,8 +41,8 @@ for (dirname, dirs, files) in os.walk(ROOT):
             exit_val = os.system(f"{PY} -m pip install -r ../requirements.txt") # install pip dependencies
             #resultCodeList.append(exit_val)
             #exit_val = os.system(f"{PY} -m unittest discover") # execute the tests
-            exit_val = os.system(f"python3.7 -m coverage run --append --omit=*/site-packages*,*/dist-packages* -m unittest discover") #TEST CODE COVERAGE
-            coverage_paths_set.add(os.path.normpath(dirname))
+            exit_val = os.system(f"{PY} -m coverage run --append --omit=*/site-packages*,*/dist-packages* -m unittest discover") #TEST CODE COVERAGE
+            microservice_coverage_paths_set.add(os.path.normpath(dirname))
             
             resultCodeList.append(exit_val) #once per folder  i.e if 3 tests are in a folder and crash, there will be just one exit val
         except Exception as e:
@@ -56,8 +60,8 @@ for (dirname, dirs, files) in os.walk(ROOT):
                     #cur_dir = os.path(dirname).parent()
                     os.chdir(cur_dir)
                     print(f"Executing coverage test in {cur_dir}")
-                    exit_val = os.system(f"python3.7 -m coverage run --append --omit=*/site-packages* -m unittest discover")
-                    coverage_paths_set.add(os.path.normpath(cur_dir))
+                    exit_val = os.system(f"{PY} -m coverage run --append --omit=*/site-packages* -m unittest discover")
+                    microservice_coverage_paths_set.add(os.path.normpath(cur_dir))
 
         except Exception as e:
             print(e)
@@ -83,7 +87,7 @@ except Exception as e:
 
 print("Combinging coverages")
 counter = 0
-for path in coverage_paths_set:
+for path in microservice_coverage_paths_set:
     try:
         path += '/.coverage'
         original = os.path.normpath( path )
@@ -96,11 +100,20 @@ for path in coverage_paths_set:
     except Exception as e:
         print(e)
         continue
+try:
+    coverage_xml_path = os.path.normpath( str(REPORTS) + '/coverage.xml')
+    os.remove(coverage_xml_path)
+
+    #coverage_html_path = os.path.normpath( str(REPORTS) + '/htmlcov' )
+    #os.rmdir(coverage_html_path)
+except Exception as e:
+    print(e)
+
 
 print("Generating Combined report")
-os.system("python3.7 -m coverage combine")
-os.system("python3.7 -m coverage xml")
-os.system("python3.7 -m coverage html") #if you want to generate the html as well
+os.system(f"{PY} -m coverage combine")
+os.system(f"{PY} -m coverage xml")
+os.system(f"{PY} -m coverage html") #if you want to generate the html as well
 
 firstError = -1
 i = 0

src/data-hub/role-stage-discovery-microservice/app/test_coverage.py

@@ -3,7 +3,9 @@ import sys
 for path in ['../', './']:
     sys.path.insert(1, path)
 
-
+#####################################
+### Don't include for test report ###
+#####################################
 try: 
     class TestCoverage(unittest.TestCase):
         def test_init_main(self):

src/data-hub/role-stage-discovery-microservice/app/tests/test_cluster.py

-try:
-    import unittest
-    import sys
-    for path in ['../', './']:
-        sys.path.insert(1, path)
 
-    # python -m unittest discover
-    from db.entities import Cluster
+import unittest
+import sys
+for path in ['../', './']:
+    sys.path.insert(1, path)
 
-    from datetime import date, datetime
-    import json
+# python -m unittest discover
+from db.entities import Cluster
 
+from datetime import date, datetime
+import json
 
-    class TestCluster(unittest.TestCase):
-        def test_init_Cluster(self):
-            try:
-                c = Cluster('debug', 'debug-table1', 'layer1', 1, [1, 2, 3])
 
-                self.assertEqual('debug', c.use_case)
-                self.assertEqual('debug-table1', c.use_case_table)
-                self.assertEqual(1, c.cluster_label)
-                self.assertEqual([1, 2, 3], c.nodes)
-            except Exception as e:
-                print(e)
+class TestCluster(unittest.TestCase):
+    def test_init_Cluster(self):
+        c = Cluster('debug', 'debug-table1', 'layer1', 1, [1, 2, 3])
 
-    if __name__ == '__main__':
-        unittest.main()
-except Exception as e:
-    print ("Exception found:")
-    print (e)
+        self.assertEqual('debug', c.use_case)
+        self.assertEqual('debug-table1', c.use_case_table)
+        self.assertEqual(1, c.cluster_label)
+        self.assertEqual([1, 2, 3], c.nodes)
+
+if __name__ == '__main__':
+    unittest.main()

src/data-hub/role-stage-discovery-microservice/app/tests/test_cluster_result.py

-try:
-    import unittest
-    import sys
-    for path in ['../', './']:
-        sys.path.insert(1, path)
-
-    # python -m unittest discover
-    from processing.clustering import ClusterResultConverter, ClusterResult
-    from typing import List, Dict, Any
-
-    class TestClusterResult(unittest.TestCase):
-        converter:ClusterResultConverter = None
-
-        def setUp(self):
-            try:
-                self.converter = ClusterResultConverter()
-            except Exception as e:
-                print (e)
-
-        def test_result_undefined_feature(self):
-            try:
-                cluster_groups = self._get_some_cluster_groups_1d()
-                cluster_res = self.converter.convert_to_cluster_results(
-                    cluster_groups=cluster_groups,
-                    features=[]
-                )
-
-                self.assert_correct_cluster_result_len(cluster_groups, cluster_res)
-                self.assert_correct_cluster_result_labels(['n.a.','n.a.','n.a.'], cluster_res)
-            except Exception as e:
-                print (e)
-
-        def test_result_1d_feature(self):
-            try:
-                cluster_groups = self._get_some_cluster_groups_1d()
-                cluster_res = self.converter.convert_to_cluster_results(
-                    cluster_groups=cluster_groups,
-                    features=['v']
-                )
-                
-
-                self.assert_correct_cluster_result_len(cluster_groups, cluster_res)
-                self.assert_correct_cluster_result_labels(['-1.0 -- 1.0','10.0 -- 11.0','2.0 -- 2.0'], cluster_res)
-
-            except Exception as e:
-                print (e)
-
-        def test_result_2d_features(self):
-
-            try:
-                cluster_groups = self._get_some_cluster_groups_2d()
-                cluster_res = self.converter.convert_to_cluster_results(
-                    cluster_groups=cluster_groups,
-                    features=['v', 'u']
-                )
-
-                self.assert_correct_cluster_result_len(cluster_groups, cluster_res)
-                self.assert_correct_cluster_result_labels([str((0.0,0.0)), str((10.5,10.5)), str((2.0,2.0)), str((3.0,6.0))], cluster_res)
-
-            except Exception as e:
-                print (e)
+import unittest
+import sys
+for path in ['../', './']:
+    sys.path.insert(1, path)
+
+# python -m unittest discover
+from processing.clustering import ClusterResultConverter, ClusterResult
+from typing import List, Dict, Any
+
+class TestClusterResult(unittest.TestCase):
+    converter:ClusterResultConverter = None
+
+    def setUp(self):
+    
+        self.converter = ClusterResultConverter()
+    
+
+    def test_result_undefined_feature(self):
+        cluster_groups = self._get_some_cluster_groups_1d()
+        cluster_res = self.converter.convert_to_cluster_results(
+            cluster_groups=cluster_groups,
+            features=[]
+        )
+
+        self.assert_correct_cluster_result_len(cluster_groups, cluster_res)
+        self.assert_correct_cluster_result_labels(['n.a.','n.a.','n.a.'], cluster_res)
+
+    def test_result_1d_feature(self):
+        cluster_groups = self._get_some_cluster_groups_1d()
+        cluster_res = self.converter.convert_to_cluster_results(
+            cluster_groups=cluster_groups,
+            features=['v']
+        )
         
 
-        #region Custom Assertions
-
-        def assert_correct_cluster_result_len(self, expected: 'original dict of lists', actual: Dict[Any, ClusterResult]):
-            try:
-                self.assertEqual(len(expected), len(actual))
-                for i in range(len(expected)):
-                    self.assertEqual(len(expected[i]), len(actual[i].nodes))
-                    self.assertEqual(expected[i], actual[i].nodes)
-
-            except Exception as e:
-                print (e)
-
-        def assert_correct_cluster_result_labels(self, expected: List[str], actual: Dict[Any, ClusterResult]):
-            try:
-                self.assertEqual(len(expected), len(actual))
-                for i in range(len(expected)):
-                    self.assertEqual(expected[i], actual[i].label)
-            except Exception as e:
-                print (e)
-
-        #endregion Custom Assertions
-
-        #region helper methods
-        def _get_some_cluster_groups_1d(self):
-            return {
-                0: [{'v':'0'}, {'v':'1'}, {'v':'-1'}],
-                1: [{'v':'10'}, {'v':'11'}],
-                2: [{'v':'2'}],
-            }
-
-        def _get_some_cluster_groups_2d(self):
-            return {
-                0: [{'v':'0', 'u':'0'}, {'v':'1', 'u':'1'}, {'v':'-1', 'u':'-1'}],
-                1: [{'v':'10', 'u':'10'}, {'v':'11', 'u':'11'}],
-                2: [{'v':'2', 'u':'2'}],
-                3: [{'v':'7', 'u':'7'}, {'v':'5', 'u':'3'}, {'v':'-3', 'u':'8'}],
-            }
-        #endregion helper methods
-
-    if __name__ == '__main__':
-        unittest.main()
-except Exception as e:
-    print ("Exception found:")
-    print (e)
+        self.assert_correct_cluster_result_len(cluster_groups, cluster_res)
+        self.assert_correct_cluster_result_labels(['-1.0 -- 1.0','10.0 -- 11.0','2.0 -- 2.0'], cluster_res)
+
+    def test_result_2d_features(self):
+
+        cluster_groups = self._get_some_cluster_groups_2d()
+        cluster_res = self.converter.convert_to_cluster_results(
+            cluster_groups=cluster_groups,
+            features=['v', 'u']
+        )
+
+        self.assert_correct_cluster_result_len(cluster_groups, cluster_res)
+        self.assert_correct_cluster_result_labels([str((0.0,0.0)), str((10.5,10.5)), str((2.0,2.0)), str((3.0,6.0))], cluster_res)
+
+    
+
+    #region Custom Assertions
+
+    def assert_correct_cluster_result_len(self, expected: 'original dict of lists', actual: Dict[Any, ClusterResult]):
+        self.assertEqual(len(expected), len(actual))
+        for i in range(len(expected)):
+            self.assertEqual(len(expected[i]), len(actual[i].nodes))
+            self.assertEqual(expected[i], actual[i].nodes)
+
+
+    def assert_correct_cluster_result_labels(self, expected: List[str], actual: Dict[Any, ClusterResult]):
+        self.assertEqual(len(expected), len(actual))
+        for i in range(len(expected)):
+            self.assertEqual(expected[i], actual[i].label)
+
+    #endregion Custom Assertions
+
+    #region helper methods
+    def _get_some_cluster_groups_1d(self):
+        return {
+            0: [{'v':'0'}, {'v':'1'}, {'v':'-1'}],
+            1: [{'v':'10'}, {'v':'11'}],
+            2: [{'v':'2'}],
+        }
+
+    def _get_some_cluster_groups_2d(self):
+        return {
+            0: [{'v':'0', 'u':'0'}, {'v':'1', 'u':'1'}, {'v':'-1', 'u':'-1'}],
+            1: [{'v':'10', 'u':'10'}, {'v':'11', 'u':'11'}],
\ No newline at end of file
+            2: [{'v':'2', 'u':'2'}],
+            3: [{'v':'7', 'u':'7'}, {'v':'5', 'u':'3'}, {'v':'-3', 'u':'8'}],
+        }
+    #endregion helper methods
+
+if __name__ == '__main__':
+    unittest.main()
\ No newline at end of file

src/data-hub/semantic-linking-microservice/app/test_coverage.py

@@ -3,7 +3,9 @@ import sys
 for path in ['../', './']:
     sys.path.insert(1, path)
 
-
+#####################################
+### Don't include for test report ###
+#####################################
 try:
     class TestCoverage(unittest.TestCase):
         def test_init_main(self):

src/data-hub/semantic-linking-microservice/app/tests/test_pipeline.py

@@ -67,8 +67,8 @@ class Test_Pipeline(unittest.TestCase):
                 }
             }
         }
-
-    def testTraceProcessing(self):
+#original name testTraceProcessing
+    def test_handle_new_trace_newTraceMsg_correctlyInserted(self):
         msg = self._buildTraceMessage()
         self.handler.handle_new_trace(msg["content"])
         self.assertEqual(len(self.handler._repository.layernodes),1)

src/participation-hub/business-logic-microservice/app/test_coverage.py

@@ -3,7 +3,9 @@ import sys
 for path in ['../', './']:
     sys.path.insert(1, path)
 
-
+#####################################
+### Don't include for test report ###
+#####################################
 try:
     class TestCoverage(unittest.TestCase):
         def test_init_main(self):

src/participation-hub/business-logic-microservice/app/tests/test_layer_adapter.py

@@ -4,7 +4,7 @@ from db.entities.layer_adapter import LayerAdapter
 
 
 class Test_Layer_Adapter(unittest.TestCase):
-    def test_valid_adapter(self):
+    def test_LayerAdapter_newLayerAdapterObj_validInstantiation(self):
         adapter1 = LayerAdapter("layer1", "use_case", "table", ["a", "c"], ["a"])
         print(adapter1.to_serializable_dict)
 

src/rest-gateway/app/test_coverage.py

@@ -2,6 +2,11 @@ import unittest
 import sys
 for path in ['../', './']:
     sys.path.insert(1, path)
+
+#####################################
+### Don't include for test report ###
+#####################################
+
 try:
     class TestCoverage(unittest.TestCase):
         def test_init_main(self):

src/transaction-hub-in/trace-retrieval-microservice/app/test_coverage.py

@@ -3,6 +3,10 @@ import sys
 for path in ['../', './']:
     sys.path.insert(1, path)
 
+#####################################
+### Don't include for test report ###
+#####################################
+
 try:
     class TestCoverage(unittest.TestCase):
         def test_init_main(self):

src/transaction-hub-in/trace-retrieval-microservice/app/tests/test_MessageHandler.py

@@ -170,7 +170,7 @@ class Test_MessageHandler(unittest.TestCase):
         self.assertEqual('semantic-linking', self.msg_sender.last_message['key'])
         self.assertEqual('new-trace', json.loads(self.msg_sender.last_message['msg'])["type"])
 
-    def test_handleblockchain_duplicateTrace(self):
+    def test_handleBlockchainTransaction_duplicateTrace_oneTransAddedToDuplicateRepo(self):
         msg = self._get_valid_message()
         msg2 = self._get_valid_message()
         msg = eval(msg)
@@ -179,7 +179,7 @@ class Test_MessageHandler(unittest.TestCase):
         self.handler.handle_blockchain_transaction(msg2['content'])
         self.assertEqual(len(self.repo.added_transactions),len(self.repo.duplicated_transactions))
 
-    def test_handleblockchain_duplicateTraceDifferentTable(self):
+    def test_handleBlockchainTransaction_duplicateTraceDifferentTable_bothTransactionsAddedAsUnique(self):
         msg = self._get_valid_message()
         msg2 = self._get_valid_message2()
         msg = eval(msg)
@@ -188,7 +188,7 @@ class Test_MessageHandler(unittest.TestCase):
         self.handler.handle_blockchain_transaction(msg2['content'])
         self.assertEqual(len(self.repo.added_transactions),2)
 
-    def test_handleblockchain_duplicateTraceDifferentUseCase(self):
+    def test_handleBlockchainTransaction_duplicateTraceDifferentUseCase_bothTransactionsAddedAsUnique(self):
         msg = self._get_valid_message()
         msg2 = self._get_valid_message3()
         msg = eval(msg)