Skip to content

S
SMART
Commit 0ba62690 authored by Manuel's avatar Manuel
Browse files
Options

Merge remote-tracking branch 'origin/staging' into bugfix/cors

parents 9921dd27 e7b081ad
Hide whitespace changes
Inline Side-by-side
Showing with 1070 additions and 676 deletions
bin/testing.py
View file @ 0ba62690
...@@ -4,7 +4,6 @@ import importlib.util ...@@ -4,7 +4,6 @@ import importlib.util
import pathlib import pathlib
import shutil import shutil
import re import re
''' '''
This script searches for all 'tests/' directories and executes all tests This script searches for all 'tests/' directories and executes all tests
by cd'ing into the dir and executing unittest discover. by cd'ing into the dir and executing unittest discover.
...@@ -12,16 +11,21 @@ It additionally installs all dependencies from a '../requirements.txt' via pip. ...@@ -12,16 +11,21 @@ It additionally installs all dependencies from a '../requirements.txt' via pip.
Use command line argument '-w' to run on windows. Use command line argument '-w' to run on windows.
''' '''
PY = sys.argv[2] if (len(sys.argv) > 1 and sys.argv[1] == '-py') else 'python' # use -py to use your own python command PY = sys.argv[2] if (len(sys.argv) > 1 and sys.argv[1] == '-py') else 'python3.7' # use -py to use your own python command
ROOT = pathlib.Path(__file__).parent.parent.absolute() ROOT = pathlib.Path(__file__).parent.parent.absolute()
REPORTS = ROOT / 'reports' REPORTS = ROOT / 'reports'
TESTS_FOLDER_NAME = os.path.normpath("/tests") TESTS_FOLDER_NAME = os.path.normpath("/tests")
print("Creating VENV")
os.system(f"{PY} -m venv venv")
PY = f"~/smart/venv/bin/{PY}"
print("\nSearching for tests at the path: "+ str(ROOT)) print("\nSearching for tests at the path: "+ str(ROOT))
count = 0 count = 0
resultCodeList = [] resultCodeList = []
coverage_paths_set = set() microservice_coverage_paths_set = set()
for (dirname, dirs, files) in os.walk(ROOT): for (dirname, dirs, files) in os.walk(ROOT):
#I assume all the tests are placed in a folder named "tests" #I assume all the tests are placed in a folder named "tests"
...@@ -37,8 +41,8 @@ for (dirname, dirs, files) in os.walk(ROOT): ...@@ -37,8 +41,8 @@ for (dirname, dirs, files) in os.walk(ROOT):
exit_val = os.system(f"{PY} -m pip install -r ../requirements.txt") # install pip dependencies exit_val = os.system(f"{PY} -m pip install -r ../requirements.txt") # install pip dependencies
#resultCodeList.append(exit_val) #resultCodeList.append(exit_val)
#exit_val = os.system(f"{PY} -m unittest discover") # execute the tests #exit_val = os.system(f"{PY} -m unittest discover") # execute the tests
exit_val = os.system(f"python3.7 -m coverage run --append --omit=*/site-packages*,*/dist-packages* -m unittest discover") #TEST CODE COVERAGE exit_val = os.system(f"{PY} -m coverage run --append --omit=*/site-packages*,*/dist-packages* -m unittest discover") #TEST CODE COVERAGE
coverage_paths_set.add(os.path.normpath(dirname)) microservice_coverage_paths_set.add(os.path.normpath(dirname))
resultCodeList.append(exit_val) #once per folder i.e if 3 tests are in a folder and crash, there will be just one exit val resultCodeList.append(exit_val) #once per folder i.e if 3 tests are in a folder and crash, there will be just one exit val
except Exception as e: except Exception as e:
...@@ -56,8 +60,8 @@ for (dirname, dirs, files) in os.walk(ROOT): ...@@ -56,8 +60,8 @@ for (dirname, dirs, files) in os.walk(ROOT):
#cur_dir = os.path(dirname).parent() #cur_dir = os.path(dirname).parent()
os.chdir(cur_dir) os.chdir(cur_dir)
print(f"Executing coverage test in {cur_dir}") print(f"Executing coverage test in {cur_dir}")
exit_val = os.system(f"python3.7 -m coverage run --append --omit=*/site-packages* -m unittest discover") exit_val = os.system(f"{PY} -m coverage run --append --omit=*/site-packages* -m unittest discover")
coverage_paths_set.add(os.path.normpath(cur_dir)) microservice_coverage_paths_set.add(os.path.normpath(cur_dir))
except Exception as e: except Exception as e:
print(e) print(e)
...@@ -83,7 +87,7 @@ except Exception as e: ...@@ -83,7 +87,7 @@ except Exception as e:
print("Combinging coverages") print("Combinging coverages")
counter = 0 counter = 0
for path in coverage_paths_set: for path in microservice_coverage_paths_set:
try: try:
path += '/.coverage' path += '/.coverage'
original = os.path.normpath( path ) original = os.path.normpath( path )
...@@ -96,11 +100,20 @@ for path in coverage_paths_set: ...@@ -96,11 +100,20 @@ for path in coverage_paths_set:
except Exception as e: except Exception as e:
print(e) print(e)
continue continue
try:
coverage_xml_path = os.path.normpath( str(REPORTS) + '/coverage.xml')
os.remove(coverage_xml_path)
#coverage_html_path = os.path.normpath( str(REPORTS) + '/htmlcov' )
#os.rmdir(coverage_html_path)
except Exception as e:
print(e)
print("Generating Combined report") print("Generating Combined report")
os.system("python3.7 -m coverage combine") os.system(f"{PY} -m coverage combine")
os.system("python3.7 -m coverage xml") os.system(f"{PY} -m coverage xml")
os.system("python3.7 -m coverage html") #if you want to generate the html as well os.system(f"{PY} -m coverage html") #if you want to generate the html as well
firstError = -1 firstError = -1
i = 0 i = 0
......
documentation/external_access.md
View file @ 0ba62690
...@@ -24,6 +24,7 @@ This token is used for authentication as _regular user_ on all microservices cur ...@@ -24,6 +24,7 @@ This token is used for authentication as _regular user_ on all microservices cur
``` ```
{ {
"ApplicationType": "use-case identifier as string", "ApplicationType": "use-case identifier as string",
"docType": "use-case-table identifier as string",
"key": "value", "key": "value",
... ...
} }
...@@ -91,4 +92,4 @@ Returns the computed similarity. Two clusters belonging to the SAME layer will b ...@@ -91,4 +92,4 @@ Returns the computed similarity. Two clusters belonging to the SAME layer will b
## Connected Cluster ## Connected Cluster
Intermediary data-structure used only by the function which computes the similarity. Clusters are connected only to other clusters belonging to a DIFFERENT layer. Intermediary data-structure used only by the function which computes the similarity. Clusters are connected only to other clusters belonging to a DIFFERENT layer.
```GET https://articonf1.itec.aau.at:30103/api/use_cases/{use_case}/tables{table}/connectedClusters``` returns all connected clusters for the given use-case and table. ```GET https://articonf1.itec.aau.at:30103/api/use_cases/{use_case}/tables/{table}/connectedClusters``` returns all connected clusters for the given use-case and table.
src/data-hub/role-stage-discovery-microservice/app/test_coverage.py
View file @ 0ba62690
...@@ -3,7 +3,9 @@ import sys ...@@ -3,7 +3,9 @@ import sys
for path in ['../', './']: for path in ['../', './']:
sys.path.insert(1, path) sys.path.insert(1, path)
#####################################
### Don't include for test report ###
#####################################
try: try:
class TestCoverage(unittest.TestCase): class TestCoverage(unittest.TestCase):
def test_init_main(self): def test_init_main(self):
......
src/data-hub/role-stage-discovery-microservice/app/tests/test_cluster.py
View file @ 0ba62690
try:
import unittest
import sys
for path in ['../', './']:
sys.path.insert(1, path)
# python -m unittest discover import unittest
from db.entities import Cluster import sys
for path in ['../', './']:
sys.path.insert(1, path)
from datetime import date, datetime # python -m unittest discover
import json from db.entities import Cluster
from datetime import date, datetime
import json
class TestCluster(unittest.TestCase):
def test_init_Cluster(self):
try:
c = Cluster('debug', 'debug-table1', 'layer1', 1, [1, 2, 3])
self.assertEqual('debug', c.use_case) class TestCluster(unittest.TestCase):
self.assertEqual('debug-table1', c.use_case_table) def test_init_Cluster(self):
self.assertEqual(1, c.cluster_label) c = Cluster('debug', 'debug-table1', 'layer1', 1, [1, 2, 3])
self.assertEqual([1, 2, 3], c.nodes)
except Exception as e:
print(e)
if __name__ == '__main__': self.assertEqual('debug', c.use_case)
unittest.main() self.assertEqual('debug-table1', c.use_case_table)
except Exception as e: self.assertEqual(1, c.cluster_label)
print ("Exception found:") self.assertEqual([1, 2, 3], c.nodes)
print (e)
if __name__ == '__main__':
unittest.main()
src/data-hub/role-stage-discovery-microservice/app/tests/test_cluster_result.py
View file @ 0ba62690
try: import unittest
import unittest import sys
import sys for path in ['../', './']:
for path in ['../', './']: sys.path.insert(1, path)
sys.path.insert(1, path)
# python -m unittest discover
# python -m unittest discover from processing.clustering import ClusterResultConverter, ClusterResult
from processing.clustering import ClusterResultConverter, ClusterResult from typing import List, Dict, Any
from typing import List, Dict, Any
class TestClusterResult(unittest.TestCase):
class TestClusterResult(unittest.TestCase): converter:ClusterResultConverter = None
converter:ClusterResultConverter = None
def setUp(self):
def setUp(self):
try: self.converter = ClusterResultConverter()
self.converter = ClusterResultConverter()
except Exception as e:
print (e) def test_result_undefined_feature(self):
cluster_groups = self._get_some_cluster_groups_1d()
def test_result_undefined_feature(self): cluster_res = self.converter.convert_to_cluster_results(
try: cluster_groups=cluster_groups,
cluster_groups = self._get_some_cluster_groups_1d() features=[]
cluster_res = self.converter.convert_to_cluster_results( )
cluster_groups=cluster_groups,
features=[] self.assert_correct_cluster_result_len(cluster_groups, cluster_res)
) self.assert_correct_cluster_result_labels(['n.a.','n.a.','n.a.'], cluster_res)
self.assert_correct_cluster_result_len(cluster_groups, cluster_res) def test_result_1d_feature(self):
self.assert_correct_cluster_result_labels(['n.a.','n.a.','n.a.'], cluster_res) cluster_groups = self._get_some_cluster_groups_1d()
except Exception as e: cluster_res = self.converter.convert_to_cluster_results(
print (e) cluster_groups=cluster_groups,
features=['v']
def test_result_1d_feature(self): )
try:
cluster_groups = self._get_some_cluster_groups_1d()
cluster_res = self.converter.convert_to_cluster_results(
cluster_groups=cluster_groups,
features=['v']
)
self.assert_correct_cluster_result_len(cluster_groups, cluster_res)
self.assert_correct_cluster_result_labels(['-1.0 -- 1.0','10.0 -- 11.0','2.0 -- 2.0'], cluster_res)
except Exception as e:
print (e)
def test_result_2d_features(self):
try:
cluster_groups = self._get_some_cluster_groups_2d()
cluster_res = self.converter.convert_to_cluster_results(
cluster_groups=cluster_groups,
features=['v', 'u']
)
self.assert_correct_cluster_result_len(cluster_groups, cluster_res)
self.assert_correct_cluster_result_labels([str((0.0,0.0)), str((10.5,10.5)), str((2.0,2.0)), str((3.0,6.0))], cluster_res)
except Exception as e:
print (e)
#region Custom Assertions self.assert_correct_cluster_result_len(cluster_groups, cluster_res)
self.assert_correct_cluster_result_labels(['-1.0 -- 1.0','10.0 -- 11.0','2.0 -- 2.0'], cluster_res)
def assert_correct_cluster_result_len(self, expected: 'original dict of lists', actual: Dict[Any, ClusterResult]):
try: def test_result_2d_features(self):
self.assertEqual(len(expected), len(actual))
for i in range(len(expected)): cluster_groups = self._get_some_cluster_groups_2d()
self.assertEqual(len(expected[i]), len(actual[i].nodes)) cluster_res = self.converter.convert_to_cluster_results(
self.assertEqual(expected[i], actual[i].nodes) cluster_groups=cluster_groups,
features=['v', 'u']
except Exception as e: )
print (e)
self.assert_correct_cluster_result_len(cluster_groups, cluster_res)
def assert_correct_cluster_result_labels(self, expected: List[str], actual: Dict[Any, ClusterResult]): self.assert_correct_cluster_result_labels([str((0.0,0.0)), str((10.5,10.5)), str((2.0,2.0)), str((3.0,6.0))], cluster_res)
try:
self.assertEqual(len(expected), len(actual))
for i in range(len(expected)):
self.assertEqual(expected[i], actual[i].label) #region Custom Assertions
except Exception as e:
print (e) def assert_correct_cluster_result_len(self, expected: 'original dict of lists', actual: Dict[Any, ClusterResult]):
self.assertEqual(len(expected), len(actual))
#endregion Custom Assertions for i in range(len(expected)):
self.assertEqual(len(expected[i]), len(actual[i].nodes))
#region helper methods self.assertEqual(expected[i], actual[i].nodes)
def _get_some_cluster_groups_1d(self):
return {
0: [{'v':'0'}, {'v':'1'}, {'v':'-1'}], def assert_correct_cluster_result_labels(self, expected: List[str], actual: Dict[Any, ClusterResult]):
1: [{'v':'10'}, {'v':'11'}], self.assertEqual(len(expected), len(actual))
2: [{'v':'2'}], for i in range(len(expected)):
} self.assertEqual(expected[i], actual[i].label)
def _get_some_cluster_groups_2d(self): #endregion Custom Assertions
return {
0: [{'v':'0', 'u':'0'}, {'v':'1', 'u':'1'}, {'v':'-1', 'u':'-1'}], #region helper methods
1: [{'v':'10', 'u':'10'}, {'v':'11', 'u':'11'}], def _get_some_cluster_groups_1d(self):
2: [{'v':'2', 'u':'2'}], return {
3: [{'v':'7', 'u':'7'}, {'v':'5', 'u':'3'}, {'v':'-3', 'u':'8'}], 0: [{'v':'0'}, {'v':'1'}, {'v':'-1'}],
} 1: [{'v':'10'}, {'v':'11'}],
#endregion helper methods 2: [{'v':'2'}],
}
if __name__ == '__main__':
unittest.main() def _get_some_cluster_groups_2d(self):
except Exception as e: return {
print ("Exception found:") 0: [{'v':'0', 'u':'0'}, {'v':'1', 'u':'1'}, {'v':'-1', 'u':'-1'}],
print (e) 1: [{'v':'10', 'u':'10'}, {'v':'11', 'u':'11'}],
\ No newline at end of file 2: [{'v':'2', 'u':'2'}],
3: [{'v':'7', 'u':'7'}, {'v':'5', 'u':'3'}, {'v':'-3', 'u':'8'}],
}
#endregion helper methods
if __name__ == '__main__':
unittest.main()
\ No newline at end of file
src/data-hub/role-stage-discovery-microservice/app/tests/test_clusterer.py
View file @ 0ba62690
try:
import unittest
import sys
for path in ['../', './']:
sys.path.insert(1, path)
import unittest
import sys
for path in ['../', './']:
sys.path.insert(1, path)
# python -m unittest discover
from processing.clustering import Clusterer, ClusterResult
import numpy as np
from typing import List, Dict, Any
class TestClusterer(unittest.TestCase): # python -m unittest discover
clusterer:Clusterer = None from processing.clustering import Clusterer, ClusterResult
import numpy as np
from typing import List, Dict, Any
def setUp(self): class TestClusterer(unittest.TestCase):
self.clusterer = Clusterer(min_points=2) clusterer:Clusterer = None
#region _extract_features def setUp(self):
self.clusterer = Clusterer(min_points=2)
def test_extract_features_emptyDataset_noResults(self): #region _extract_features
features = self.clusterer._extract_features(dataset=[], features=['test'])
np.testing.assert_equal(np.asarray([]), features)
def test_extract_features_emptyFeatures_singleEmptyResult(self): def test_extract_features_emptyDataset_noResults(self):
features = self.clusterer._extract_features(dataset=[{'a':1, 'b':2}], features=[]) features = self.clusterer._extract_features(dataset=[], features=['test'])
np.testing.assert_equal(np.asarray([[]]), features) np.testing.assert_equal(np.asarray([]), features)
def test_extract_features_singleFeature_Projection(self): def test_extract_features_emptyFeatures_singleEmptyResult(self):
features = self.clusterer._extract_features(dataset=[{'a':1, 'b':2}], features=['a']) features = self.clusterer._extract_features(dataset=[{'a':1, 'b':2}], features=[])
np.testing.assert_equal(np.asarray([[1]]), features) np.testing.assert_equal(np.asarray([[]]), features)
def test_extract_features_singleFeature_Projection_2(self): def test_extract_features_singleFeature_Projection(self):
features = self.clusterer._extract_features(dataset=[{'a':1, 'b':2}, {'a':3, 'b':4}], features=['a']) features = self.clusterer._extract_features(dataset=[{'a':1, 'b':2}], features=['a'])
np.testing.assert_equal(np.asarray([[1], [3]]), features) np.testing.assert_equal(np.asarray([[1]]), features)
def test_extract_features_multFeature_Projection(self): def test_extract_features_singleFeature_Projection_2(self):
features = self.clusterer._extract_features(dataset=[{'a':0, 'b':2, 'c':4}, {'a':1, 'b':3, 'c':5}], features=['a','c']) features = self.clusterer._extract_features(dataset=[{'a':1, 'b':2}, {'a':3, 'b':4}], features=['a'])
np.testing.assert_equal(np.asarray([[0,4], [1,5]]), features) np.testing.assert_equal(np.asarray([[1], [3]]), features)
#endregion _extract_features def test_extract_features_multFeature_Projection(self):
features = self.clusterer._extract_features(dataset=[{'a':0, 'b':2, 'c':4}, {'a':1, 'b':3, 'c':5}], features=['a','c'])
np.testing.assert_equal(np.asarray([[0,4], [1,5]]), features)
#endregion _extract_features
#region create_labels #region create_labels
def test_create_labels_noneInput_noneOutput(self): def test_create_labels_noneInput_noneOutput(self):
labels = self.clusterer.create_labels(None) labels = self.clusterer.create_labels(None)
self.assertEqual(None, labels) self.assertEqual(None, labels)
def test_create_labels_emptyInput_emptyOutput(self): def test_create_labels_emptyInput_emptyOutput(self):
labels = self.clusterer.create_labels([]) labels = self.clusterer.create_labels([])
self.assertEqual([], labels) self.assertEqual([], labels)
def test_create_labels_singleInput_noise(self): def test_create_labels_singleInput_noise(self):
clusterer = Clusterer(min_points=1) clusterer = Clusterer(min_points=1)
features = clusterer._extract_features(dataset=[self.location(1,2)], features=self.get_location_features()) features = clusterer._extract_features(dataset=[self.location(1,2)], features=self.get_location_features())
labels = clusterer.create_labels(features) labels = clusterer.create_labels(features)
self.assertEqual(1, len(labels)) self.assertEqual(1, len(labels))
self.assertEqual(-1, labels[0]) self.assertEqual(-1, labels[0])
def test_create_labels_tooSmallInputForMinPtsHyperparameter_error(self): def test_create_labels_tooSmallInputForMinPtsHyperparameter_error(self):
clusterer = Clusterer(min_points=3) clusterer = Clusterer(min_points=3)
features = clusterer._extract_features(dataset=[self.location(1,2), self.location(1,2)], features=self.get_location_features()) features = clusterer._extract_features(dataset=[self.location(1,2), self.location(1,2)], features=self.get_location_features())
with self.assertRaises(ValueError): with self.assertRaises(ValueError):
# Fails because (min_pts > |input elements|) # Fails because (min_pts > |input elements|)
clusterer.create_labels(features) clusterer.create_labels(features)
def test_create_labels_nearInputs_singleCluster(self): def test_create_labels_nearInputs_singleCluster(self):
locations = [self.location(1,2), self.location(2,2)] locations = [self.location(1,2), self.location(2,2)]
features = self.clusterer._extract_features(dataset=locations, features=self.get_location_features()) features = self.clusterer._extract_features(dataset=locations, features=self.get_location_features())
labels = self.clusterer.create_labels(features) labels = self.clusterer.create_labels(features)
self.assertEqual(2, len(labels)) self.assertEqual(2, len(labels))
self.assertEqual(labels[0], labels[1]) self.assertEqual(labels[0], labels[1])
def test_create_labels_nearInputs_twoClusters(self): def test_create_labels_nearInputs_twoClusters(self):
locations = [self.location(1,2), self.location(2,2), self.location(20,20), self.location(20,23)] locations = [self.location(1,2), self.location(2,2), self.location(20,20), self.location(20,23)]
features = self.clusterer._extract_features(dataset=locations, features=self.get_location_features()) features = self.clusterer._extract_features(dataset=locations, features=self.get_location_features())
labels = self.clusterer.create_labels(features) labels = self.clusterer.create_labels(features)
self.assertEqual(4, len(labels)) self.assertEqual(4, len(labels))
self.assertEqual(labels[0], labels[1]) self.assertEqual(labels[0], labels[1])
self.assertEqual(labels[2], labels[3]) self.assertEqual(labels[2], labels[3])
self.assertNotEqual(labels[0], labels[2]) self.assertNotEqual(labels[0], labels[2])
#endregion create_labels #endregion create_labels
#region label_dataset #region label_dataset
def test_label_dataset_NoneLocations_NoException(self): def test_label_dataset_NoneLocations_NoException(self):
self.clusterer.label_dataset(None, []) self.clusterer.label_dataset(None, [])
def test_label_dataset_NoneLabels_NoException(self): def test_label_dataset_NoneLabels_NoException(self):
self.clusterer.label_dataset([], None) self.clusterer.label_dataset([], None)
def test_label_dataset_emptyInput_emptyOutput(self): def test_label_dataset_emptyInput_emptyOutput(self):
locations = [] locations = []
self.clusterer.label_dataset(locations, []) self.clusterer.label_dataset(locations, [])
self.assertEqual(0, len(locations)) self.assertEqual(0, len(locations))
def test_label_dataset_diffInputLengths_ValueError_1(self): def test_label_dataset_diffInputLengths_ValueError_1(self):
with self.assertRaises(ValueError): with self.assertRaises(ValueError):
self.clusterer.label_dataset([], [1]) self.clusterer.label_dataset([], [1])
def test_label_dataset_diffInputLengths_ValueError_2(self): def test_label_dataset_diffInputLengths_ValueError_2(self):
with self.assertRaises(ValueError): with self.assertRaises(ValueError):
self.clusterer.label_dataset([self.location(1,2)], []) self.clusterer.label_dataset([self.location(1,2)], [])
def test_label_dataset_multInput_correctlyLabeled(self): def test_label_dataset_multInput_correctlyLabeled(self):
locations = [self.location(1,2), self.location(2,2), self.location(20,20)] locations = [self.location(1,2), self.location(2,2), self.location(20,20)]
labels = [17,2,20] labels = [17,2,20]
self.clusterer.label_dataset(locations, labels) self.clusterer.label_dataset(locations, labels)
self.assertEqual(3, len(locations)) self.assertEqual(3, len(locations))
self.assertHaveLabelsAsNewKey(locations, labels) self.assertHaveLabelsAsNewKey(locations, labels)
#endregion label_dataset #endregion label_dataset
#region cluster_dataset #region cluster_dataset
def test_cluster_dataset_locationsMultInput_correctlyLabeled(self): def test_cluster_dataset_locationsMultInput_correctlyLabeled(self):
locations = [self.location(1,2), self.location(2,2), self.location(20,20), self.location(20,21)] locations = [self.location(1,2), self.location(2,2), self.location(20,20), self.location(20,21)]
labels = [0,0,1,1] labels = [0,0,1,1]
exp_res = {0:locations[0:2], 1:locations[2:4]} exp_res = {0:locations[0:2], 1:locations[2:4]}
res = self.clusterer.cluster_dataset(locations, self.get_location_features()) res = self.clusterer.cluster_dataset(locations, self.get_location_features())
self.assertHaveLabelsAsNewKey(locations, labels) self.assertHaveLabelsAsNewKey(locations, labels)
self.assertClusteringResult(exp_res, res) self.assertClusteringResult(exp_res, res)
def test_cluster_dataset_timesMultInput_correctlyLabeled(self): def test_cluster_dataset_timesMultInput_correctlyLabeled(self):
times = [self.time(123), self.time(128), self.time(223), self.time(225)] times = [self.time(123), self.time(128), self.time(223), self.time(225)]
labels = [0,0,1,1] labels = [0,0,1,1]
exp_res = {0:times[0:2], 1:times[2:4]} exp_res = {0:times[0:2], 1:times[2:4]}
res = self.clusterer.cluster_dataset(times, self.get_time_features()) res = self.clusterer.cluster_dataset(times, self.get_time_features())
self.assertHaveLabelsAsNewKey(times, labels) self.assertHaveLabelsAsNewKey(times, labels)
self.assertClusteringResult(exp_res, res) self.assertClusteringResult(exp_res, res)
def test_cluster_dataset_locationsMultInput_correctlyLabeled_2(self): def test_cluster_dataset_locationsMultInput_correctlyLabeled_2(self):
clusterer = Clusterer(3) clusterer = Clusterer(3)
locations = [ locations = [
self.location(1,2), self.location(2,2), self.location(2,2), self.location(1,2), self.location(2,2), self.location(2,2),
self.location(20,20), self.location(20,21), self.location(20,20), self.location(20,20), self.location(20,21), self.location(20,20),
self.location(50,50), self.location(50,50),
self.location(50,1), self.location(50,2), self.location(50,-1) self.location(50,1), self.location(50,2), self.location(50,-1)
] ]
labels = [0,0,0,1,1,1,-1,2,2,2] labels = [0,0,0,1,1,1,-1,2,2,2]
exp_res = {0:locations[0:3], 1:locations[3:6], -1:locations[6:7], 2:locations[7:10]} exp_res = {0:locations[0:3], 1:locations[3:6], -1:locations[6:7], 2:locations[7:10]}
res = clusterer.cluster_dataset(locations, self.get_location_features()) res = clusterer.cluster_dataset(locations, self.get_location_features())
self.assertHaveLabelsAsNewKey(locations, labels) self.assertHaveLabelsAsNewKey(locations, labels)
self.assertClusteringResult(exp_res, res) self.assertClusteringResult(exp_res, res)
#endregion cluster_dataset #endregion cluster_dataset
#region helper methods #region helper methods
def location(self, lat, long_) -> dict: def location(self, lat, long_) -> dict:
return {'latitude': lat, 'longitude':long_} return {'latitude': lat, 'longitude':long_}
def get_location_features(self): def get_location_features(self):
return ['latitude', 'longitude'] return ['latitude', 'longitude']
def time(self, ts) -> dict: def time(self, ts) -> dict:
return {'timestamp': ts} return {'timestamp': ts}
def get_time_features(self): def get_time_features(self):
return ['timestamp'] return ['timestamp']
def assertHaveLabelsAsNewKey(self, locations, labels): def assertHaveLabelsAsNewKey(self, locations, labels):
self.assertEqual(len(labels), len(locations)) self.assertEqual(len(labels), len(locations))
for i in range(len(locations)): for i in range(len(locations)):
self.assertEqual(labels[i], locations[i]['cluster_label']) self.assertEqual(labels[i], locations[i]['cluster_label'])
def assertClusteringResult(self, expected: Dict[Any, List], actual: Dict[Any, ClusterResult]): def assertClusteringResult(self, expected: Dict[Any, List], actual: Dict[Any, ClusterResult]):
self.assertEqual(len(expected), len(actual)) self.assertEqual(len(expected), len(actual))
for k in expected.keys(): for k in expected.keys():
if k not in actual: if k not in actual:
self.fail(f"Cluster key ({k}, {type(k)}) not in result.") self.fail(f"Cluster key ({k}, {type(k)}) not in result.")
self.assertListEqual(expected[k], actual[k].nodes) self.assertListEqual(expected[k], actual[k].nodes)
#endregion helper methods #endregion helper methods
if __name__ == '__main__':
unittest.main()
if __name__ == '__main__':
unittest.main()
except Exception as e:
print ("Exception found:")
print (e)
src/data-hub/role-stage-discovery-microservice/app/tests/test_similarity.py
View file @ 0ba62690
try: import unittest
import unittest
import sys import sys
for path in ['../', './', '../../../modules/', '../../../../modules']: for path in ['../', './', '../../../modules/', '../../../../modules']:
sys.path.insert(1, path) sys.path.insert(1, path)
from db.entities.connected_node import NodeC from db.entities.connected_node import NodeC
from db.entities.connected_cluster import ClusterC from db.entities.connected_cluster import ClusterC
from db.entities.connected_layer import LayerC from db.entities.connected_layer import LayerC
import math import math
import datetime import datetime
from typing import Dict from typing import Dict
from typing import Dict from typing import Dict
from processing.similarityFiles.dataInput import * from processing.similarityFiles.dataInput import *
from processing.similarityFiles.calculateWeights import * from processing.similarityFiles.calculateWeights import *
from processing.similarityFiles.calculateSimilarity import * from processing.similarityFiles.calculateSimilarity import *
from processing.similarityFiles.miscFunctions import * from processing.similarityFiles.miscFunctions import *
import json import json
class TestSimilarity(unittest.TestCase): class TestSimilarity(unittest.TestCase):
'''Tests the similarity calculation which works without object orientation.''' '''Tests the similarity calculation which works without object orientation.'''
def test_integration_similarityCalculation(self): def test_integration_calculateSimilarity_ClustersDict_CorrectValue(self):
''' '''
Only for testing, can be deleted at any time.\n Only for testing, can be deleted at any time.\n
Served as a testing example to make sure the computations are correct Served as a testing example to make sure the computations are correct
''' '''
limitNrNodes = 100000 limitNrNodes = 100000
layerDict = dict() layerDict = dict()
("Creating Connected_cluster dict and similarity dict") ("Creating Connected_cluster dict and similarity dict")
inputLayerLocation=getTestLocationLayerData() inputLayerLocation=getTestLocationLayerData()
inputLayerTime=getTestTimeLayerData() inputLayerTime=getTestTimeLayerData()
inputLayerPrice=getTestPriceLayerData() inputLayerPrice=getTestPriceLayerData()
layerDict = populateWithNewNodesSingleLayer(inputLayerLocation,layerDict,limitNrNodes) layerDict = populateWithNewNodesSingleLayer(inputLayerLocation,layerDict,limitNrNodes)
layerDict = populateWithNewNodesSingleLayer(inputLayerTime,layerDict,limitNrNodes) layerDict = populateWithNewNodesSingleLayer(inputLayerTime,layerDict,limitNrNodes)
layerDict = populateWithNewNodesSingleLayer(inputLayerPrice,layerDict,limitNrNodes) layerDict = populateWithNewNodesSingleLayer(inputLayerPrice,layerDict,limitNrNodes)
layerDict = calculateWeights(layerDict) layerDict = calculateWeights(layerDict)
similarityDict = calculateSimilarity(layerDict) similarityDict = calculateSimilarity(layerDict)
("Asserting if the calculated values are true") ("Asserting if the calculated values are true")
#assert x == "hello" #assert x == "hello"
# SYNTAX: # SYNTAX:
#similarityDict[(clusterLabel1,clusterLabel2,layerOfTheClusters)][layerToWhichTheClustersAreCompared] == ExpectedSimilarityValue #similarityDict[(clusterLabel1,clusterLabel2,layerOfTheClusters)][layerToWhichTheClustersAreCompared] == ExpectedSimilarityValue
#checking if the cluster "1" and cluster "2" from the "Location" layer have the sqrt(2) similarity values for when compared with 'Price'and 'Time'layers #checking if the cluster "1" and cluster "2" from the "Location" layer have the sqrt(2) similarity values for when compared with 'Price'and 'Time'layers
self.assertEqual(similarityDict[(1,2,'Location')]['Price'], math.sqrt(2)) self.assertEqual(similarityDict[(1,2,'Location')]['Price'], math.sqrt(2))
self.assertEqual(similarityDict[(1,2,'Location')]['Time'], math.sqrt(2)) self.assertEqual(similarityDict[(1,2,'Location')]['Time'], math.sqrt(2))
self.assertEqual(similarityDict[(1,3,'Location')]['Price'], math.sqrt(10)) self.assertEqual(similarityDict[(1,3,'Location')]['Price'], math.sqrt(10))
self.assertEqual(similarityDict[(1,3,'Location')]['Time'], math.sqrt(16)) self.assertEqual(similarityDict[(1,3,'Location')]['Time'], math.sqrt(16))
self.assertEqual(similarityDict[(2,3,'Location')]['Price'], math.sqrt(4)) self.assertEqual(similarityDict[(2,3,'Location')]['Price'], math.sqrt(4))
self.assertEqual(similarityDict[(2,3,'Location')]['Time'], math.sqrt(10)) self.assertEqual(similarityDict[(2,3,'Location')]['Time'], math.sqrt(10))
self.assertEqual(similarityDict[(4,5,'Time')]['Location'], math.sqrt(19)) self.assertEqual(similarityDict[(4,5,'Time')]['Location'], math.sqrt(19))
self.assertEqual(similarityDict[(4,5,'Time')]['Price'], math.sqrt(26)) self.assertEqual(similarityDict[(4,5,'Time')]['Price'], math.sqrt(26))
self.assertEqual(similarityDict[(6,7,'Price')]['Location'], math.sqrt(3)) self.assertEqual(similarityDict[(6,7,'Price')]['Location'], math.sqrt(3))
self.assertEqual(similarityDict[(6,7,'Price')]['Time'], math.sqrt(8)) self.assertEqual(similarityDict[(6,7,'Price')]['Time'], math.sqrt(8))
# assert similarityDict[(1,2,'Location')] # assert similarityDict[(1,2,'Location')]
("Test Passed Succesfully") ("Test Passed Succesfully")
return layerDict return layerDict
def getTestLocationLayerData(): def getTestLocationLayerData():
inputLayerLocation = [ inputLayerLocation = [
{
"cluster_label": 1,
"layer_name": "Location",
"nodes": [
{
"Finished_time": 1576631193265951,
"Latitude_Destination": -5.973257,
"Longitude_Destination": 37.416316,
"TravelID": "asd",
"TravelPrice": 15,
"UniqueID": "a",
"UserID": "asdf"
},
{
"Finished_time": 1576631193265951,
"Latitude_Destination": -5.973257,
"Longitude_Destination": 37.416316,
"TravelID": "5e57ec9159bc0668543f156d",
"TravelPrice": 15,
"UniqueID": "aa",
"UserID": "a95075f5042b1b27060080156d87fe34ec7e712c"
},
{
"Finished_time": 1576631193265951,
"Latitude_Destination": -5.973257,
"Longitude_Destination": 37.416316,
"TravelID": "asd",
"TravelPrice": 15,
"UniqueID": "aa",
"UserID": "asdf"
},
{
"Finished_time": 1576631193265951,
"Latitude_Destination": -5.973257,
"Longitude_Destination": 37.416316,
"TravelID": "asd",
"TravelPrice": 15,
"UniqueID": "aaa",
"UserID": "asdf"
},
{
"Finished_time": 1576631193265951,
"Latitude_Destination": -5.973257,
"Longitude_Destination": 37.416316,
"TravelID": "5e57ec9159bc0668543f156d",
"TravelPrice": 15,
"UniqueID": "aaaa",
"UserID": "a95075f5042b1b27060080156d87fe34ec7e712c"
}
]
},
{
"cluster_label": 2,
"layer_name": "Location",
"nodes": [
{
"Finished_time": 1576631193265951,
"Latitude_Destination": -5.973257,
"Longitude_Destination": 37.416316,
"TravelID": "5e57ec9159bc0668543f156b",
"TravelPrice": 15,
"UniqueID": "b",
"UserID": "a95075f5042b1b27060080156d87fe34ec7e712c"
},
{
"Finished_time": 1576631193265951,
"Latitude_Destination": -5.973257,
"Longitude_Destination": 37.416316,
"TravelID": "asd",
"TravelPrice": 15,
"UniqueID": "bb",
"UserID": "asdf"
},
{
"Finished_time": 1576631193265951,
"Latitude_Destination": -5.973257,
"Longitude_Destination": 37.416316,
"TravelID": "asd",
"TravelPrice": 15,
"UniqueID": "bbb",
"UserID": "asdf"
},
{
"Finished_time": 1576631193265951,
"Latitude_Destination": -5.973257,
"Longitude_Destination": 37.416316,
"TravelID": "5e57ec9159bc0668543f156d",
"TravelPrice": 15,
"UniqueID": "bb",
"UserID": "a95075f5042b1b27060080156d87fe34ec7e712c"
}
]
},
{
"cluster_label": 3,
"layer_name": "Location",
"nodes": [
{
"Finished_time": 1576631193265951,
"Latitude_Destination": -5.973257,
"Longitude_Destination": 37.416316,
"TravelID": "5e57ec9159bc0668543f156c",
"TravelPrice": 15,
"UniqueID": "c",
"UserID": "a95075f5042b1b27060080156d87fe34ec7e712c"
},
{
"Finished_time": 1576631193265951,
"Latitude_Destination": -5.973257,
"Longitude_Destination": 37.416316,
"TravelID": "asd",
"TravelPrice": 15,
"UniqueID": "c",
"UserID": "asdf"
},
{
"Finished_time": 1576631193265951,
"Latitude_Destination": -5.973257,
"Longitude_Destination": 37.416316,
"TravelID": "asd",
"TravelPrice": 15,
"UniqueID": "cc",
"UserID": "asdf"
}
]
}
]
return inputLayerLocation
def getTestTimeLayerData():
inputLayerTime = [
{ {
"cluster_label": 4, "cluster_label": 1,
"layer_name": "Time", "layer_name": "Location",
"nodes": [ "nodes": [
{ {
"Finished_time": 1576631193265951, "Finished_time": 1576631193265951,
"Latitude_Destination": -5.973257, "Latitude_Destination": -5.973257,
"Longitude_Destination": 37.416316, "Longitude_Destination": 37.416316,
"TravelID": "5e57ec9159bc0668543f156d", "TravelID": "asd",
"TravelPrice": 15, "TravelPrice": 15,
"UniqueID": "a", "UniqueID": "a",
"UserID": "a95075f5042b1b27060080156d87fe34ec7e712c" "UserID": "asdf"
}, },
{ {
"Finished_time": 1576631193265951, "Finished_time": 1576631193265951,
...@@ -227,37 +95,19 @@ try: ...@@ -227,37 +95,19 @@ try:
"Finished_time": 1576631193265951, "Finished_time": 1576631193265951,
"Latitude_Destination": -5.973257, "Latitude_Destination": -5.973257,
"Longitude_Destination": 37.416316, "Longitude_Destination": 37.416316,
"TravelID": "5e57ec9159bc0668543f156d", "TravelID": "asd",
"TravelPrice": 15,
"UniqueID": "aaa",
"UserID": "a95075f5042b1b27060080156d87fe34ec7e712c"
},
{
"Finished_time": 1576631193265951,
"Latitude_Destination": -5.973257,
"Longitude_Destination": 37.416316,
"TravelID": "5e57ec9159bc0668543f156d",
"TravelPrice": 15, "TravelPrice": 15,
"UniqueID": "aaaa", "UniqueID": "aa",
"UserID": "a95075f5042b1b27060080156d87fe34ec7e712c" "UserID": "asdf"
},
{
"Finished_time": 1576631193265951,
"Latitude_Destination": -5.973257,
"Longitude_Destination": 37.416316,
"TravelID": "5e57ec9159bc0668543f156d",
"TravelPrice": 15,
"UniqueID": "b",
"UserID": "a95075f5042b1b27060080156d87fe34ec7e712c"
}, },
{ {
"Finished_time": 1576631193265951, "Finished_time": 1576631193265951,
"Latitude_Destination": -5.973257, "Latitude_Destination": -5.973257,
"Longitude_Destination": 37.416316, "Longitude_Destination": 37.416316,
"TravelID": "5e57ec9159bc0668543f156d", "TravelID": "asd",
"TravelPrice": 15, "TravelPrice": 15,
"UniqueID": "bb", "UniqueID": "aaa",
"UserID": "a95075f5042b1b27060080156d87fe34ec7e712c" "UserID": "asdf"
}, },
{ {
"Finished_time": 1576631193265951, "Finished_time": 1576631193265951,
...@@ -265,50 +115,20 @@ try: ...@@ -265,50 +115,20 @@ try:
"Longitude_Destination": 37.416316, "Longitude_Destination": 37.416316,
"TravelID": "5e57ec9159bc0668543f156d", "TravelID": "5e57ec9159bc0668543f156d",
"TravelPrice": 15, "TravelPrice": 15,
"UniqueID": "bbb", "UniqueID": "aaaa",
"UserID": "a95075f5042b1b27060080156d87fe34ec7e712c" "UserID": "a95075f5042b1b27060080156d87fe34ec7e712c"
} }
] ]
}, },
{ {
"cluster_label": 5, "cluster_label": 2,
"layer_name": "Time", "layer_name": "Location",
"nodes": [ "nodes": [
{ {
"Finished_time": 1576631193265951, "Finished_time": 1576631193265951,
"Latitude_Destination": -5.973257, "Latitude_Destination": -5.973257,
"Longitude_Destination": 37.416316, "Longitude_Destination": 37.416316,
"TravelID": "5e57ec9159bc0668543f156f", "TravelID": "5e57ec9159bc0668543f156b",
"TravelPrice": 15,
"UniqueID": "a",
"UserID": "a95075f5042b1b27060080156d87fe34ec7e712c"
},
{
"Finished_time": 1576631193265951,
"Latitude_Destination": -5.973257,
"Longitude_Destination": 37.416316,
"TravelID": "5e57ec9159bc0668543f156d",
"TravelPrice": 15,
"UniqueID": "c",
"UserID": "a95075f5042b1b27060080156d87fe34ec7e712c"
}
]
}
]
return inputLayerTime
def getTestPriceLayerData():
inputLayerPrice = [
{
"cluster_label": 6,
"layer_name": "Price",
"nodes": [
{
"Finished_time": 1576631193265951,
"Latitude_Destination": -5.973257,
"Longitude_Destination": 37.416316,
"TravelID": "5e57ec9159bc0668543f156h",
"TravelPrice": 15, "TravelPrice": 15,
"UniqueID": "b", "UniqueID": "b",
"UserID": "a95075f5042b1b27060080156d87fe34ec7e712c" "UserID": "a95075f5042b1b27060080156d87fe34ec7e712c"
...@@ -317,27 +137,19 @@ try: ...@@ -317,27 +137,19 @@ try:
"Finished_time": 1576631193265951, "Finished_time": 1576631193265951,
"Latitude_Destination": -5.973257, "Latitude_Destination": -5.973257,
"Longitude_Destination": 37.416316, "Longitude_Destination": 37.416316,
"TravelID": "5e57ec9159bc0668543f156d", "TravelID": "asd",
"TravelPrice": 15, "TravelPrice": 15,
"UniqueID": "bb", "UniqueID": "bb",
"UserID": "a95075f5042b1b27060080156d87fe34ec7e712c" "UserID": "asdf"
},{
"Finished_time": 1576631193265951,
"Latitude_Destination": -5.973257,
"Longitude_Destination": 37.416316,
"TravelID": "5e57ec9159bc0668543f156d",
"TravelPrice": 15,
"UniqueID": "aa",
"UserID": "a95075f5042b1b27060080156d87fe34ec7e712c"
}, },
{ {
"Finished_time": 1576631193265951, "Finished_time": 1576631193265951,
"Latitude_Destination": -5.973257, "Latitude_Destination": -5.973257,
"Longitude_Destination": 37.416316, "Longitude_Destination": 37.416316,
"TravelID": "5e57ec9159bc0668543f156d", "TravelID": "asd",
"TravelPrice": 15, "TravelPrice": 15,
"UniqueID": "aaa", "UniqueID": "bbb",
"UserID": "a95075f5042b1b27060080156d87fe34ec7e712c" "UserID": "asdf"
}, },
{ {
"Finished_time": 1576631193265951, "Finished_time": 1576631193265951,
...@@ -345,61 +157,245 @@ try: ...@@ -345,61 +157,245 @@ try:
"Longitude_Destination": 37.416316, "Longitude_Destination": 37.416316,
"TravelID": "5e57ec9159bc0668543f156d", "TravelID": "5e57ec9159bc0668543f156d",
"TravelPrice": 15, "TravelPrice": 15,
"UniqueID": "aaaa", "UniqueID": "bb",
"UserID": "a95075f5042b1b27060080156d87fe34ec7e712c" "UserID": "a95075f5042b1b27060080156d87fe34ec7e712c"
} }
] ]
}, },
{ {
"cluster_label": 7, "cluster_label": 3,
"layer_name": "Price", "layer_name": "Location",
"nodes": [ "nodes": [
{ {
"Finished_time": 1576631193265951, "Finished_time": 1576631193265951,
"Latitude_Destination": -5.973257, "Latitude_Destination": -5.973257,
"Longitude_Destination": 37.416316, "Longitude_Destination": 37.416316,
"TravelID": "5e57ec9159bc0668543f156g", "TravelID": "5e57ec9159bc0668543f156c",
"TravelPrice": 15,
"UniqueID": "a",
"UserID": "a95075f5042b1b27060080156d87fe34ec7e712c"
},
{
"Finished_time": 1576631193265951,
"Latitude_Destination": -5.973257,
"Longitude_Destination": 37.416316,
"TravelID": "5e57ec9159bc0668543f156d",
"TravelPrice": 15, "TravelPrice": 15,
"UniqueID": "aa", "UniqueID": "c",
"UserID": "a95075f5042b1b27060080156d87fe34ec7e712c" "UserID": "a95075f5042b1b27060080156d87fe34ec7e712c"
}, },
{ {
"Finished_time": 1576631193265951, "Finished_time": 1576631193265951,
"Latitude_Destination": -5.973257, "Latitude_Destination": -5.973257,
"Longitude_Destination": 37.416316, "Longitude_Destination": 37.416316,
"TravelID": "5e57ec9159bc0668543f156d", "TravelID": "asd",
"TravelPrice": 15, "TravelPrice": 15,
"UniqueID": "b", "UniqueID": "c",
"UserID": "a95075f5042b1b27060080156d87fe34ec7e712c" "UserID": "asdf"
}, },
{ {
"Finished_time": 1576631193265951, "Finished_time": 1576631193265951,
"Latitude_Destination": -5.973257, "Latitude_Destination": -5.973257,
"Longitude_Destination": 37.416316, "Longitude_Destination": 37.416316,
"TravelID": "5e57ec9159bc0668543f156d", "TravelID": "asd",
"TravelPrice": 15, "TravelPrice": 15,
"UniqueID": "c", "UniqueID": "cc",
"UserID": "a95075f5042b1b27060080156d87fe34ec7e712c" "UserID": "asdf"
} }
] ]
} }
] ]
return inputLayerPrice return inputLayerLocation
def getTestTimeLayerData():
inputLayerTime = [
{
"cluster_label": 4,
"layer_name": "Time",
"nodes": [
{
"Finished_time": 1576631193265951,
"Latitude_Destination": -5.973257,
"Longitude_Destination": 37.416316,
"TravelID": "5e57ec9159bc0668543f156d",
"TravelPrice": 15,
"UniqueID": "a",
"UserID": "a95075f5042b1b27060080156d87fe34ec7e712c"
},
{
"Finished_time": 1576631193265951,
"Latitude_Destination": -5.973257,
"Longitude_Destination": 37.416316,
"TravelID": "5e57ec9159bc0668543f156d",
"TravelPrice": 15,
"UniqueID": "aa",
"UserID": "a95075f5042b1b27060080156d87fe34ec7e712c"
},
{
"Finished_time": 1576631193265951,
"Latitude_Destination": -5.973257,
"Longitude_Destination": 37.416316,
"TravelID": "5e57ec9159bc0668543f156d",
"TravelPrice": 15,
"UniqueID": "aaa",
"UserID": "a95075f5042b1b27060080156d87fe34ec7e712c"
},
{
"Finished_time": 1576631193265951,
"Latitude_Destination": -5.973257,
"Longitude_Destination": 37.416316,
"TravelID": "5e57ec9159bc0668543f156d",
"TravelPrice": 15,
"UniqueID": "aaaa",
"UserID": "a95075f5042b1b27060080156d87fe34ec7e712c"
},
{
"Finished_time": 1576631193265951,
"Latitude_Destination": -5.973257,
"Longitude_Destination": 37.416316,
"TravelID": "5e57ec9159bc0668543f156d",
"TravelPrice": 15,
"UniqueID": "b",
"UserID": "a95075f5042b1b27060080156d87fe34ec7e712c"
},
{
"Finished_time": 1576631193265951,
"Latitude_Destination": -5.973257,
"Longitude_Destination": 37.416316,
"TravelID": "5e57ec9159bc0668543f156d",
"TravelPrice": 15,
"UniqueID": "bb",
"UserID": "a95075f5042b1b27060080156d87fe34ec7e712c"
},
{
"Finished_time": 1576631193265951,
"Latitude_Destination": -5.973257,
"Longitude_Destination": 37.416316,
"TravelID": "5e57ec9159bc0668543f156d",
"TravelPrice": 15,
"UniqueID": "bbb",
"UserID": "a95075f5042b1b27060080156d87fe34ec7e712c"
}
]
},
{
"cluster_label": 5,
"layer_name": "Time",
"nodes": [
{
"Finished_time": 1576631193265951,
"Latitude_Destination": -5.973257,
"Longitude_Destination": 37.416316,
"TravelID": "5e57ec9159bc0668543f156f",
"TravelPrice": 15,
"UniqueID": "a",
"UserID": "a95075f5042b1b27060080156d87fe34ec7e712c"
},
{
"Finished_time": 1576631193265951,
"Latitude_Destination": -5.973257,
"Longitude_Destination": 37.416316,
"TravelID": "5e57ec9159bc0668543f156d",
"TravelPrice": 15,
"UniqueID": "c",
"UserID": "a95075f5042b1b27060080156d87fe34ec7e712c"
}
]
}
]
return inputLayerTime
def getTestPriceLayerData():
inputLayerPrice = [
{
"cluster_label": 6,
"layer_name": "Price",
"nodes": [
{
"Finished_time": 1576631193265951,
"Latitude_Destination": -5.973257,
"Longitude_Destination": 37.416316,
"TravelID": "5e57ec9159bc0668543f156h",
"TravelPrice": 15,
"UniqueID": "b",
"UserID": "a95075f5042b1b27060080156d87fe34ec7e712c"
},
{
"Finished_time": 1576631193265951,
"Latitude_Destination": -5.973257,
"Longitude_Destination": 37.416316,
"TravelID": "5e57ec9159bc0668543f156d",
"TravelPrice": 15,
"UniqueID": "bb",
"UserID": "a95075f5042b1b27060080156d87fe34ec7e712c"
},{
"Finished_time": 1576631193265951,
"Latitude_Destination": -5.973257,
"Longitude_Destination": 37.416316,
"TravelID": "5e57ec9159bc0668543f156d",
"TravelPrice": 15,
"UniqueID": "aa",
"UserID": "a95075f5042b1b27060080156d87fe34ec7e712c"
},
{
"Finished_time": 1576631193265951,
"Latitude_Destination": -5.973257,
"Longitude_Destination": 37.416316,
"TravelID": "5e57ec9159bc0668543f156d",
"TravelPrice": 15,
"UniqueID": "aaa",
"UserID": "a95075f5042b1b27060080156d87fe34ec7e712c"
},
{
"Finished_time": 1576631193265951,
"Latitude_Destination": -5.973257,
"Longitude_Destination": 37.416316,
"TravelID": "5e57ec9159bc0668543f156d",
"TravelPrice": 15,
"UniqueID": "aaaa",
"UserID": "a95075f5042b1b27060080156d87fe34ec7e712c"
}
]
},
{
"cluster_label": 7,
"layer_name": "Price",
"nodes": [
{
"Finished_time": 1576631193265951,
"Latitude_Destination": -5.973257,
"Longitude_Destination": 37.416316,
"TravelID": "5e57ec9159bc0668543f156g",
"TravelPrice": 15,
"UniqueID": "a",
"UserID": "a95075f5042b1b27060080156d87fe34ec7e712c"
},
{
"Finished_time": 1576631193265951,
"Latitude_Destination": -5.973257,
"Longitude_Destination": 37.416316,
"TravelID": "5e57ec9159bc0668543f156d",
"TravelPrice": 15,
"UniqueID": "aa",
"UserID": "a95075f5042b1b27060080156d87fe34ec7e712c"
},
{
"Finished_time": 1576631193265951,
"Latitude_Destination": -5.973257,
"Longitude_Destination": 37.416316,
"TravelID": "5e57ec9159bc0668543f156d",
"TravelPrice": 15,
"UniqueID": "b",
"UserID": "a95075f5042b1b27060080156d87fe34ec7e712c"
},
{
"Finished_time": 1576631193265951,
"Latitude_Destination": -5.973257,
"Longitude_Destination": 37.416316,
"TravelID": "5e57ec9159bc0668543f156d",
"TravelPrice": 15,
"UniqueID": "c",
"UserID": "a95075f5042b1b27060080156d87fe34ec7e712c"
}
]
}
]
return inputLayerPrice
if __name__ == '__main__': if __name__ == '__main__':
unittest.main() unittest.main()
except Exception as e: \ No newline at end of file
print ("Exception found:")
print (e)
\ No newline at end of file
src/data-hub/semantic-linking-microservice/app/main.py
View file @ 0ba62690
...@@ -20,6 +20,7 @@ from env_info import is_running_locally, get_resources_path ...@@ -20,6 +20,7 @@ from env_info import is_running_locally, get_resources_path
from messaging.ReconnectingMessageManager import ReconnectingMessageManager from messaging.ReconnectingMessageManager import ReconnectingMessageManager
from messaging.MessageHandler import MessageHandler from messaging.MessageHandler import MessageHandler
from flask import request from flask import request
from flask_cors import CORS
from flask import redirect from flask import redirect
# init message handler # init message handler
...@@ -30,6 +31,7 @@ def message_received_callback(channel, method, properties, body): ...@@ -30,6 +31,7 @@ def message_received_callback(channel, method, properties, body):
# load swagger config # load swagger config
app = connexion.App(__name__, specification_dir='configs/') app = connexion.App(__name__, specification_dir='configs/')
CORS(app.app)
@app.app.before_request @app.app.before_request
def before_request(): def before_request():
......
src/data-hub/semantic-linking-microservice/app/requirements.txt
View file @ 0ba62690
...@@ -10,6 +10,7 @@ connexion==2.7.0 ...@@ -10,6 +10,7 @@ connexion==2.7.0
coverage==5.3.1 coverage==5.3.1
cryptography==3.1 cryptography==3.1
Flask==1.1.2 Flask==1.1.2
Flask-Cors==3.0.10
idna==2.9 idna==2.9
importlib-metadata==1.6.1 importlib-metadata==1.6.1
inflection==0.5.0 inflection==0.5.0
......
src/data-hub/semantic-linking-microservice/app/test_coverage.py
View file @ 0ba62690
...@@ -3,7 +3,9 @@ import sys ...@@ -3,7 +3,9 @@ import sys
for path in ['../', './']: for path in ['../', './']:
sys.path.insert(1, path) sys.path.insert(1, path)
#####################################
### Don't include for test report ###
#####################################
try: try:
class TestCoverage(unittest.TestCase): class TestCoverage(unittest.TestCase):
def test_init_main(self): def test_init_main(self):
......
src/data-hub/semantic-linking-microservice/app/tests/test_pipeline.py
View file @ 0ba62690
...@@ -67,8 +67,8 @@ class Test_Pipeline(unittest.TestCase): ...@@ -67,8 +67,8 @@ class Test_Pipeline(unittest.TestCase):
} }
} }
} }
#original name testTraceProcessing
def testTraceProcessing(self): def test_handle_new_trace_newTraceMsg_correctlyInserted(self):
msg = self._buildTraceMessage() msg = self._buildTraceMessage()
self.handler.handle_new_trace(msg["content"]) self.handler.handle_new_trace(msg["content"])
self.assertEqual(len(self.handler._repository.layernodes),1) self.assertEqual(len(self.handler._repository.layernodes),1)
......
src/participation-hub/business-logic-microservice/app/_add_use_case_scripts/vialog/add_vialog_schema.py
View file @ 0ba62690
...@@ -9,7 +9,7 @@ for modules_path in modules_paths: ...@@ -9,7 +9,7 @@ for modules_path in modules_paths:
if os.path.exists(modules_path): if os.path.exists(modules_path):
sys.path.insert(1, modules_path) sys.path.insert(1, modules_path)
from _add_use_case_scripts.vialog.tables import add_user, add_video from _add_use_case_scripts.vialog.tables import add_user, add_video, add_change
import network_constants as nc import network_constants as nc
from security.token_manager import TokenManager from security.token_manager import TokenManager
...@@ -38,4 +38,4 @@ if __name__ == "__main__": ...@@ -38,4 +38,4 @@ if __name__ == "__main__":
add_use_case(use_case) add_use_case(use_case)
add_user.main(use_case) add_user.main(use_case)
add_video.main(use_case) add_video.main(use_case)
add_change.main(use_case)
\ No newline at end of file \ No newline at end of file
src/participation-hub/business-logic-microservice/app/_add_use_case_scripts/vialog/add_vialog_schema_new_enum.py 0 → 100644
View file @ 0ba62690
import sys
import os
from pathlib import Path
from typing import Dict, Any
import requests
modules_paths = ['.', '../../../modules/']
for modules_path in modules_paths:
if os.path.exists(modules_path):
sys.path.insert(1, modules_path)
from _add_use_case_scripts.vialog.tables import add_user, add_video, add_change
import network_constants as nc
from security.token_manager import TokenManager
def add_use_case(use_case: str):
#use_case = "vialog"
jwt = TokenManager.getInstance().getToken()
url = f"https://articonf1.itec.aau.at:30420/api/use-cases"
response = requests.post(
url,
verify=False,
proxies = { "http":None, "https":None },
headers = { "Authorization": f"Bearer {jwt}"},
json = {"name": use_case}
)
print(url+": "+str(response.content))
if __name__ == "__main__":
use_case = "vialog-new-enum"
# disable ssl warnings :)
requests.packages.urllib3.disable_warnings()
add_use_case(use_case)
add_video.main(use_case)
add_change.main(use_case)
\ No newline at end of file
src/participation-hub/business-logic-microservice/app/_add_use_case_scripts/vialog/tables/add_change.py 0 → 100644
View file @ 0ba62690
from _add_use_case_scripts.requestPost import postLayersToSwagger, postTableToSwagger
def add_table(use_case: str, table_name: str):
'''
take the columns and add the mappings at the server
replace all "/"'s in the internal representation with a "_"
'''
columns = [
"changeType",
"changedValue",
"previousValue",
"newValue",
]
columns = { c : c for c in columns }
columns["UniqueID"] = "videoId+changeId"
columns["changeType"] = "enum(changeType)"
columns["changedValue"] = "enum(changedValue)"
table = {
"name": table_name,
"mappings": columns
}
postTableToSwagger(use_case,table)
def add_layers(use_case:str, table_name: str):
layers = [
{
"use_case": use_case,
"table": table_name,
"name": "Changetype_Layer",
"properties": [
"changeType",
],
"cluster_properties": [
"changeType",
]
},
{
"use_case": use_case,
"table": table_name,
"name": "Changedvalue_Layer",
"properties": [
"changedValue",
"previousValue",
"newValue"
],
"cluster_properties": [
"changedValue"
]
}
]
postLayersToSwagger(use_case,layers)
def main(use_case: str):
print("user")
table_name = "change"
add_table(use_case,table_name)
add_layers(use_case,table_name)
\ No newline at end of file
src/participation-hub/business-logic-microservice/app/_add_use_case_scripts/vialog/tables/add_video.py
View file @ 0ba62690
...@@ -6,27 +6,32 @@ def add_table(use_case: str, table_name: str): ...@@ -6,27 +6,32 @@ def add_table(use_case: str, table_name: str):
''' '''
columns = [ columns = [
# "docType", 'videoId',
"videoId", 'objType',
"Video_Token", 'eventName',
"replyTo", 'video_token',
"Created", 'replyTo',
"Duration", 'created',
"videoResolution", 'duration',
"Label", 'videoResolution',
"ThreadId", 'label',
"Position", 'threadId',
"ModifiedDate", 'position',
"Views", 'views',
"ModeratedBy", 'moderatedBy',
"CommunityManagerNotes", 'moderationDate',
"Rewards", 'communityManagerNotes',
"Video_State", 'rewards',
"Video_Type" 'video_state',
'video_type'
] ]
columns = { c : c for c in columns } columns = { c : c for c in columns }
columns["UniqueID"] = "videoId" columns["UniqueID"] = "videoId"
columns["moderatedBy"] = "enum(moderatedBy)"
columns["video_state"] = "enum(video_state)"
columns["video_type"] = "enum(video_type)"
columns["videoResolution"] = "enum(videoResolution)"
table = { table = {
"name": table_name, "name": table_name,
...@@ -43,14 +48,14 @@ def add_layers(use_case:str, table_name: str): ...@@ -43,14 +48,14 @@ def add_layers(use_case:str, table_name: str):
"name": "Manager_Layer", "name": "Manager_Layer",
"properties": [ "properties": [
"UniqueID", "UniqueID",
"ModifiedDate", "moderationDate",
"ModeratedBy", "moderatedBy",
"Video_State", "video_state",
"Video_Type" "video_type"
], ],
"cluster_properties": [ "cluster_properties": [
"ModeratedBy", "moderatedBy",
"Video_State" "video_state"
] ]
}, },
{ {
...@@ -59,16 +64,16 @@ def add_layers(use_case:str, table_name: str): ...@@ -59,16 +64,16 @@ def add_layers(use_case:str, table_name: str):
"name": "Video_Popularity_Layer", "name": "Video_Popularity_Layer",
"properties": [ "properties": [
"UniqueID", "UniqueID",
"Label", "label",
"Created", "created",
"Views", "views",
"Rewards", "rewards",
"Video_State", "video_state",
"Video_Type" "video_type"
], ],
"cluster_properties": [ "cluster_properties": [
"Views", "views",
"Video_Type" "video_type"
] ]
}, },
{ {
...@@ -77,15 +82,15 @@ def add_layers(use_case:str, table_name: str): ...@@ -77,15 +82,15 @@ def add_layers(use_case:str, table_name: str):
"name": "Video_Age_Layer", "name": "Video_Age_Layer",
"properties": [ "properties": [
"UniqueID", "UniqueID",
"Label", "label",
"Created", "created",
"Views", "views",
"Rewards", "rewards",
"Video_State", "video_state",
"Video_Type" "video_type"
], ],
"cluster_properties": [ "cluster_properties": [
"Created" "created"
] ]
}, },
{ {
...@@ -94,15 +99,15 @@ def add_layers(use_case:str, table_name: str): ...@@ -94,15 +99,15 @@ def add_layers(use_case:str, table_name: str):
"name": "Rewards_Layer", "name": "Rewards_Layer",
"properties": [ "properties": [
"UniqueID", "UniqueID",
"Label", "label",
"Created", "created",
"Views", "views",
"Rewards", "rewards",
"Video_State", "video_state",
"Video_Type" "video_type"
], ],
"cluster_properties": [ "cluster_properties": [
"Rewards" "rewards"
] ]
}, },
{ {
...@@ -111,14 +116,14 @@ def add_layers(use_case:str, table_name: str): ...@@ -111,14 +116,14 @@ def add_layers(use_case:str, table_name: str):
"name": "Video_Lenght_Layer", "name": "Video_Lenght_Layer",
"properties": [ "properties": [
"UniqueID", "UniqueID",
"Created", "created",
"Views", "views",
"Duration", "duration",
"Video_State", "video_state",
"Video_Type" "video_type"
], ],
"cluster_properties": [ "cluster_properties": [
"Duration" "duration"
] ]
}, },
{ {
...@@ -127,11 +132,11 @@ def add_layers(use_case:str, table_name: str): ...@@ -127,11 +132,11 @@ def add_layers(use_case:str, table_name: str):
"name": "Video_Resolution_Layer", "name": "Video_Resolution_Layer",
"properties": [ "properties": [
"UniqueID", "UniqueID",
"Created", "created",
"Views", "views",
"videoResolution", "videoResolution",
"Video_State", "video_state",
"Video_Type" "video_type"
], ],
"cluster_properties": [ "cluster_properties": [
"videoResolution" "videoResolution"
......
src/participation-hub/business-logic-microservice/app/test_coverage.py
View file @ 0ba62690
...@@ -3,7 +3,9 @@ import sys ...@@ -3,7 +3,9 @@ import sys
for path in ['../', './']: for path in ['../', './']:
sys.path.insert(1, path) sys.path.insert(1, path)
#####################################
### Don't include for test report ###
#####################################
try: try:
class TestCoverage(unittest.TestCase): class TestCoverage(unittest.TestCase):
def test_init_main(self): def test_init_main(self):
......
src/participation-hub/business-logic-microservice/app/tests/test_layer_adapter.py
View file @ 0ba62690
...@@ -4,7 +4,7 @@ from db.entities.layer_adapter import LayerAdapter ...@@ -4,7 +4,7 @@ from db.entities.layer_adapter import LayerAdapter
class Test_Layer_Adapter(unittest.TestCase): class Test_Layer_Adapter(unittest.TestCase):
def test_valid_adapter(self): def test_LayerAdapter_newLayerAdapterObj_validInstantiation(self):
adapter1 = LayerAdapter("layer1", "use_case", "table", ["a", "c"], ["a"]) adapter1 = LayerAdapter("layer1", "use_case", "table", ["a", "c"], ["a"])
print(adapter1.to_serializable_dict) print(adapter1.to_serializable_dict)
......
src/rest-gateway/app/pull_video_data.py 0 → 100644
View file @ 0ba62690
import requests
import json
videoListUrl = "https://dev758755.vialog.app/Videos/Meta/ListAll"
videoUrl = "https://dev758755.vialog.app/stat/events?type=video&id="
# token from Rest Gateway to authorize
JWT_TOKEN = 'eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJ1c2VybmFtZSI6InJlZ3VsYXJAaXRlYy5hYXUuYXQiLCJjcmVhdGVkX2F0IjoiMjAyMS0wNC0wNyAxMjo0OTo0MS43MTkzNjQiLCJ2YWxpZF91bnRpbCI6IjIwMjEtMDQtMDggMTI6NDk6NDEuNzE5MzY0In0.FN6qqBQeJSmXtS0-0dBiL-ojz6Ou7E5Tc9macrrhM4A'
useCase = "vialog-new-enum"
def send_transaction_to_rest_gateway(transaction: dict):
res = requests.post(
url = 'https://articonf1.itec.aau.at:30401/api/trace',
json = transaction,
headers = {"Authorization": f"Bearer {JWT_TOKEN}"},
verify = False # ignore ssl error
)
print(res)
videosRequest = requests.get(videoListUrl)
empty = set()
html = set()
if (videosRequest.status_code != 200):
print(f"Status: {videosRequest.status_code}")
dataCount = 0
for video in videosRequest.json():
dataCount += 1
id = video["videoId"]
videoRequest = requests.get(f"{videoUrl}{id}")
if videoRequest.status_code != 200:
print(f"Status: {videoRequest.status_code}")
if videoRequest.text != "" and not videoRequest.text.startswith("<!DOCTYPE html>"):
print(f"\n\n{videoUrl}{id}")
historyList = sorted(videoRequest.json()[0]["History"], key=lambda k: k['moderationDate'], reverse=True)
historyList.append(empty)
i = 0
changeList = []
for eventMap in historyList:
if historyList[i+1] == empty:
break
if i == 0:
lastState = eventMap
lastState["ApplicationType"] = useCase
lastState["docType"] = "video"
print(f"{lastState}\n\n\n")
send_transaction_to_rest_gateway(lastState)
if eventMap["eventName"] == 'r1eabcbdc8f5378b2ba71a1b6fe2038b Created' or eventMap["eventName"] == 'Created':
change = {"changeType": "Created", "changedValue": "video_state", "previousValue": "", "newValue":"Created"}
elif eventMap["eventName"] == "Restore":
change = {"changeType": "Restore", "changedValue": "", "previousValue": "", "newValue":""}
elif eventMap["eventName"] == "PositionChange":
change = {"changeType": "PositionChange", "changedValue": "position", "previousValue": historyList[i+1]["position"], "newValue": eventMap["position"]}
elif eventMap["eventName"] == "Hide":
change = {"changeType": "Hide", "changedValue": "video_state", "previousValue": historyList[i+1]["video_state"], "newValue": eventMap["video_state"]}
elif eventMap["eventName"] == "Publish":
change = {"changeType": "Publish", "changedValue": "video_state", "previousValue": historyList[i+1]["video_state"], "newValue":eventMap["video_state"]}
elif eventMap["eventName"] == "CMNote":
change = {"changeType": "CMNote", "changedValue": "communityManagerNotes", "previousValue": historyList[i+1]["communityManagerNotes"], "newValue":eventMap["communityManagerNotes"]}
elif eventMap["eventName"] == "Move":
change = {"changeType": "Move", "changedValue": "position", "previousValue": historyList[i+1]["position"], "newValue":eventMap["position"]}
elif eventMap["eventName"] == "VideoType":
change = {"changeType": "VideoType", "changedValue": "video_type", "previousValue": historyList[i+1]["video_type"], "newValue":eventMap["video_type"]}
elif eventMap["eventName"] == "Delete":
change = {"changeType": "Delete", "changedValue": "", "previousValue": "", "newValue": ""}
elif eventMap["eventName"] == "Copy":
change = {"changeType": "Copy", "changedValue": "", "previousValue": "", "newValue":""}
elif eventMap["eventName"] == "CustomLabel":
change = {"changeType": "CustomLabel", "changedValue": "label", "previousValue": historyList[i+1]["label"], "newValue":eventMap["label"]}
change["videoId"] = id
change["changeId"] = i
change["timestamp"] = eventMap["moderationDate"]
change["ApplicationType"] = useCase
change["docType"] = "change"
print(change)
send_transaction_to_rest_gateway(change)
i += 1
elif videoRequest.text == "":
empty.add(id)
else:
html.add(id)
print(f"empty: {empty}\n\n")
print(f"html page: {html}\n\n")
print(f"history: {historyList}")
print(dataCount)
src/rest-gateway/app/test_coverage.py
View file @ 0ba62690
...@@ -2,6 +2,11 @@ import unittest ...@@ -2,6 +2,11 @@ import unittest
import sys import sys
for path in ['../', './']: for path in ['../', './']:
sys.path.insert(1, path) sys.path.insert(1, path)
#####################################
### Don't include for test report ###
#####################################
try: try:
class TestCoverage(unittest.TestCase): class TestCoverage(unittest.TestCase):
def test_init_main(self): def test_init_main(self):
......
src/transaction-hub-in/trace-retrieval-microservice/app/database/repository.py
View file @ 0ba62690
...@@ -31,7 +31,9 @@ class Repository(MongoRepositoryBase): ...@@ -31,7 +31,9 @@ class Repository(MongoRepositoryBase):
@throws @throws
KeyError - Duplicate transaction ID KeyError - Duplicate transaction ID
''' '''
reference = self.get_transaction_with_id(transaction.id()) use_case= transaction.use_case
table = transaction.table
reference = self.get_transaction_with_id(transaction.id(),use_case, table)
if reference == None: if reference == None:
super().insert_entry(self._transaction_collection, transaction.to_serializable_dict()) super().insert_entry(self._transaction_collection, transaction.to_serializable_dict())
else: else:
...@@ -41,10 +43,10 @@ class Repository(MongoRepositoryBase): ...@@ -41,10 +43,10 @@ class Repository(MongoRepositoryBase):
result = super().get_entries(self._transaction_collection, projection={'_id': False}, selection={"use_case": use_case}) result = super().get_entries(self._transaction_collection, projection={'_id': False}, selection={"use_case": use_case})
return [Transaction.from_serializable_dict(row) for row in list(result)] return [Transaction.from_serializable_dict(row) for row in list(result)]
def get_transaction_with_id(self, unique_id: str) -> Transaction: def get_transaction_with_id(self, unique_id: str, use_case:str, table:str ) -> Transaction:
result = list(super().get_entries(self._transaction_collection, projection={'_id': False}, selection={"UniqueID": unique_id})) result = list(super().get_entries(self._transaction_collection, projection={'_id': False}, selection={"id": unique_id,"use_case": use_case, "table":table}))
if len(result) == 1: if len(result) >= 1:
return Transaction.from_serializable_dict(result[0]) return Transaction.from_serializable_dict(result[0])
return None return None
......
src/transaction-hub-in/trace-retrieval-microservice/app/messaging/MessageHandler.py
View file @ 0ba62690
...@@ -8,9 +8,11 @@ import json ...@@ -8,9 +8,11 @@ import json
import hashlib import hashlib
import logging import logging
import requests import requests
requests.packages.urllib3.disable_warnings()
from typing import Dict from typing import Dict
LOGGER = logging.getLogger(__name__) LOGGER = logging.getLogger(__name__)
class MessageHandler: class MessageHandler:
...@@ -30,7 +32,7 @@ class MessageHandler: ...@@ -30,7 +32,7 @@ class MessageHandler:
self._rest_fetcher = rest_fetcher self._rest_fetcher = rest_fetcher
def handle_generic(self, body): def handle_generic(self, body):
LOGGER.info(f"Received message: {body}")
result = None result = None
message = None message = None
...@@ -39,11 +41,13 @@ class MessageHandler: ...@@ -39,11 +41,13 @@ class MessageHandler:
except (ValueError, TypeError): except (ValueError, TypeError):
result = self.MSG_NOT_JSON result = self.MSG_NOT_JSON
LOGGER.warning(result) LOGGER.warning(result)
LOGGER.info(f"Received message: {body}")
return result return result
if not 'type' in message: if not 'type' in message:
result = self.MSG_NO_TYPE result = self.MSG_NO_TYPE
LOGGER.warning(result) LOGGER.warning(result)
LOGGER.info(f"Received message: {body}")
return result return result
if message['type'] == 'blockchain-transaction': if message['type'] == 'blockchain-transaction':
...@@ -51,8 +55,11 @@ class MessageHandler: ...@@ -51,8 +55,11 @@ class MessageHandler:
result = self.MSG_TRACE_PROCESSED result = self.MSG_TRACE_PROCESSED
else: else:
result = self.MSG_NOT_PROCESSED result = self.MSG_NOT_PROCESSED
LOGGER.warning(result)
LOGGER.info(f"Received message: {body}")
LOGGER.info(result) #LOGGER.info(result) #too much spam
return result return result
def _resolve_path(self, data: Dict, path:str) -> Dict: def _resolve_path(self, data: Dict, path:str) -> Dict:
...@@ -132,39 +139,39 @@ class MessageHandler: ...@@ -132,39 +139,39 @@ class MessageHandler:
''' '''
# check if there is a use-case in the message # check if there is a use-case in the message
if "ApplicationType" not in transaction_message.keys(): if "ApplicationType" not in transaction_message:
LOGGER.error("Transaction has no ApplicationType, storing it under use-case 'unknown'.") LOGGER.error("Transaction has no ApplicationType, storing it under use-case 'unknown'.")
transaction_message["ApplicationType"] = "unknown" transaction_message["ApplicationType"] = "unknown"
self._mongo_repo.add_failed_transaction(transaction_message) self._mongo_repo.add_failed_transaction(transaction_message)
return return
# check if there is a doctype in the message # check if there is a table in the message
if "docType" not in transaction_message.keys(): if "docType" not in transaction_message:
LOGGER.error("Transaction has no docType, storing it under docType 'unknown'.") LOGGER.error("Transaction has no docType, storing it under table 'unknown'.")
transaction_message["docType"] = "unknown" transaction_message["docType"] = "unknown"
self._mongo_repo.add_failed_transaction(transaction_message) self._mongo_repo.add_failed_transaction(transaction_message)
return return
use_case = transaction_message["ApplicationType"] transaction_use_case = transaction_message["ApplicationType"]
docType = transaction_message["docType"] transaction_table = transaction_message["docType"]
try: try:
tables = self._rest_fetcher.fetch_schema_information(use_case) tables = self._rest_fetcher.fetch_schema_information(transaction_use_case)
except ValueError as e: except ValueError as e:
LOGGER.error(f"{e}\nStoring it as a failed transaction.") LOGGER.error(f"{e}\nCould not fetch schema, storing it as a failed transaction..")
self._mongo_repo.add_failed_transaction(transaction_message) self._mongo_repo.add_failed_transaction(transaction_message)
return return
target_table = None target_table = None
# find correct table # find correct table
for table in tables: for table in tables:
if table["name"] == docType: if table["name"] == transaction_table:
target_table = table target_table = table
break break
# abort if table does not exist. # abort if table does not exist.
if target_table == None: if target_table == None:
LOGGER.error(f"There is no table '{docType}', storing it as a failed transaction.") LOGGER.error(f"There is no table '{transaction_table}', storing it as a failed transaction.")
self._mongo_repo.add_failed_transaction(transaction_message) self._mongo_repo.add_failed_transaction(transaction_message)
return return
...@@ -172,20 +179,19 @@ class MessageHandler: ...@@ -172,20 +179,19 @@ class MessageHandler:
try: try:
flattened = self._flatten_transaction(transaction_message, mappings) flattened = self._flatten_transaction(transaction_message, mappings)
except KeyError as e: except KeyError as e:
LOGGER.error(f"Failed while flattening with KeyError: {str(e)}") LOGGER.error(f"Failed while flattening with KeyError: {str(e)}, storing it as a failed transaction.")
self._mongo_repo.add_failed_transaction(transaction_message) self._mongo_repo.add_failed_transaction(transaction_message)
return return
transaction = Transaction(use_case, target_table["name"], flattened) transaction = Transaction(transaction_use_case, target_table["name"], flattened)
#check for duplicates #check for duplicates
try: try:
reference = self._mongo_repo.get_transaction_with_id(transaction.id()) references = self._mongo_repo.get_transaction_with_id(transaction.id(),transaction_use_case,transaction_table)
if reference != None: if references != None:
if (reference[0].table == transaction.table) and (reference[0].use_case == transaction.use_case): LOGGER.info("Found duplicate, storing it as a duplicated transaction.")
LOGGER.error("Found duplicate") self._mongo_repo.add_duplicated_transaction(transaction)
self._mongo_repo.add_duplicated_transaction(transaction) return
return
except ValueError as e: except ValueError as e:
LOGGER.error(f"{e}, could not insert duplicated node.") LOGGER.error(f"{e}, could not insert duplicated node.")
return return
...@@ -193,14 +199,14 @@ class MessageHandler: ...@@ -193,14 +199,14 @@ class MessageHandler:
try: try:
self._mongo_repo.add_transaction(transaction) self._mongo_repo.add_transaction(transaction)
except KeyError as e: except KeyError as e:
LOGGER.error(f"{e}") LOGGER.error(f"{e}, ignored {transaction_message}")
self._mongo_repo.add_failed_transaction(transaction_message) # self._mongo_repo.add_failed_transaction(transaction_message)
return return
msg = { msg = {
"type": "new-trace", "type": "new-trace",
"content": transaction.to_serializable_dict(), "content": transaction.to_serializable_dict()
} }
msg_string = json.dumps(msg) msg_string = json.dumps(msg)
......
src/transaction-hub-in/trace-retrieval-microservice/app/routes/transactions.py
View file @ 0ba62690
...@@ -23,7 +23,8 @@ def delete_all_failed_for_use_case(use_case: str): ...@@ -23,7 +23,8 @@ def delete_all_failed_for_use_case(use_case: str):
return Response(status=200) return Response(status=200)
def all_duplicated_for_use_case(use_case: str): def all_duplicated_for_use_case(use_case: str):
return _repository.all_duplicated_transactions_for_use_case(use_case) transactions = _repository.all_duplicated_transactions_for_use_case(use_case)
return [t.to_serializable_dict() for t in transactions]
def delete_all_duplicated_for_use_case(use_case: str): def delete_all_duplicated_for_use_case(use_case: str):
_repository.delete_all_duplicated_transactions(use_case) _repository.delete_all_duplicated_transactions(use_case)
......
src/transaction-hub-in/trace-retrieval-microservice/app/test_coverage.py
View file @ 0ba62690
...@@ -3,6 +3,10 @@ import sys ...@@ -3,6 +3,10 @@ import sys
for path in ['../', './']: for path in ['../', './']:
sys.path.insert(1, path) sys.path.insert(1, path)
#####################################
### Don't include for test report ###
#####################################
try: try:
class TestCoverage(unittest.TestCase): class TestCoverage(unittest.TestCase):
def test_init_main(self): def test_init_main(self):
......
src/transaction-hub-in/trace-retrieval-microservice/app/tests/test_MessageHandler.py
View file @ 0ba62690
...@@ -18,14 +18,14 @@ class DummyMongoRepo: ...@@ -18,14 +18,14 @@ class DummyMongoRepo:
def add_transaction(self, transaction): def add_transaction(self, transaction):
self.added_transactions.append(transaction) self.added_transactions.append(transaction)
def get_transaction_with_id(self, unique_id: str): def get_transaction_with_id(self, unique_id: str, use_case,table):
result = [] result = []
for trans in self.added_transactions: for trans in self.added_transactions:
transID = trans.id() transID = trans.id()
if transID == unique_id: if transID == unique_id and trans.use_case == use_case and trans.table == table:
result.append(trans) result.append(trans)
if len(result) > 0: if len(result) > 0:
return result return result[0]
return None return None
...@@ -170,7 +170,7 @@ class Test_MessageHandler(unittest.TestCase): ...@@ -170,7 +170,7 @@ class Test_MessageHandler(unittest.TestCase):
self.assertEqual('semantic-linking', self.msg_sender.last_message['key']) self.assertEqual('semantic-linking', self.msg_sender.last_message['key'])
self.assertEqual('new-trace', json.loads(self.msg_sender.last_message['msg'])["type"]) self.assertEqual('new-trace', json.loads(self.msg_sender.last_message['msg'])["type"])
def test_handleblockchain_duplicateTrace(self): def test_handleBlockchainTransaction_duplicateTrace_oneTransAddedToDuplicateRepo(self):
msg = self._get_valid_message() msg = self._get_valid_message()
msg2 = self._get_valid_message() msg2 = self._get_valid_message()
msg = eval(msg) msg = eval(msg)
...@@ -178,8 +178,9 @@ class Test_MessageHandler(unittest.TestCase): ...@@ -178,8 +178,9 @@ class Test_MessageHandler(unittest.TestCase):
self.handler.handle_blockchain_transaction(msg['content']) self.handler.handle_blockchain_transaction(msg['content'])
self.handler.handle_blockchain_transaction(msg2['content']) self.handler.handle_blockchain_transaction(msg2['content'])
self.assertEqual(len(self.repo.added_transactions),len(self.repo.duplicated_transactions)) self.assertEqual(len(self.repo.added_transactions),len(self.repo.duplicated_transactions))
self.assertEqual(len(self.repo.added_transactions),1)
def test_handleblockchain_duplicateTraceDifferentTable(self): def test_handleBlockchainTransaction_duplicateTraceDifferentTable_bothTransactionsAddedAsUnique(self):
msg = self._get_valid_message() msg = self._get_valid_message()
msg2 = self._get_valid_message2() msg2 = self._get_valid_message2()
msg = eval(msg) msg = eval(msg)
...@@ -188,7 +189,7 @@ class Test_MessageHandler(unittest.TestCase): ...@@ -188,7 +189,7 @@ class Test_MessageHandler(unittest.TestCase):
self.handler.handle_blockchain_transaction(msg2['content']) self.handler.handle_blockchain_transaction(msg2['content'])
self.assertEqual(len(self.repo.added_transactions),2) self.assertEqual(len(self.repo.added_transactions),2)
def test_handleblockchain_duplicateTraceDifferentUseCase(self): def test_handleBlockchainTransaction_duplicateTraceDifferentUseCase_bothTransactionsAddedAsUnique(self):
msg = self._get_valid_message() msg = self._get_valid_message()
msg2 = self._get_valid_message3() msg2 = self._get_valid_message3()
msg = eval(msg) msg = eval(msg)
...@@ -197,5 +198,24 @@ class Test_MessageHandler(unittest.TestCase): ...@@ -197,5 +198,24 @@ class Test_MessageHandler(unittest.TestCase):
self.handler.handle_blockchain_transaction(msg2['content']) self.handler.handle_blockchain_transaction(msg2['content'])
self.assertEqual(len(self.repo.added_transactions),2) self.assertEqual(len(self.repo.added_transactions),2)
def test_handleBlockchainTransaction_multipleTransactions3SameIdDiffUseCaseTable_3AddedUnique2Duplicate(self):
#print("Entered Test: 3Unique 2Dupli")
msg = self._get_valid_message()
msg2 = self._get_valid_message2()
msg3 = self._get_valid_message3()
msg4 = self._get_valid_message3()
msg5 = self._get_valid_message3()
msg = eval(msg)
msg2 = eval(msg2)
msg3 = eval(msg3)
msg4 = eval(msg4)
msg5 = eval(msg5)
self.handler.handle_blockchain_transaction(msg['content'])
self.handler.handle_blockchain_transaction(msg2['content'])
self.handler.handle_blockchain_transaction(msg3['content'])
self.handler.handle_blockchain_transaction(msg4['content'])
self.handler.handle_blockchain_transaction(msg5['content'])
self.assertEqual(len(self.repo.added_transactions),3)
self.assertEqual(len(self.repo.duplicated_transactions),2)
if __name__ == '__main__': if __name__ == '__main__':
unittest.main() unittest.main()
\ No newline at end of file
tools/check-use-case-data/check_mog.py 0 → 100644
View file @ 0ba62690
import requests
requests.packages.urllib3.disable_warnings()
from icecream import ic
def httpget(url):
token = 'eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJ1c2VybmFtZSI6InJlZ3VsYXJAaXRlYy5hYXUuYXQiLCJjcmVhdGVkX2F0IjoiMjAyMS0wMy0yNCAxMDoxMzo1MS4wMjkwNDkiLCJ2YWxpZF91bnRpbCI6IjIwMjEtMDMtMjUgMTA6MTM6NTEuMDI5MDQ5In0.V6kYV5Lmb_tUIsF-6AKNB8_lIifmJP_Dm8gHhGa5w_o'
res = requests.get(url,
verify=False,
headers = { "Authorization": f"Bearer {token}"})
return res
# list tables
res = httpget(url = 'https://articonf1.itec.aau.at:30420/api/use-cases/crowd-journalism/tables')
print("Tables: ", [entry['name'] for entry in res.json()])
# count pushed data
def count_data(json_res, table_identifier='table'):
tables = {}
for entry in json_res:
key = entry[table_identifier]
if key not in tables:
tables[key] = 0
tables[key] += 1
ic(tables)
res = httpget(url = 'https://articonf1.itec.aau.at:30001/api/use_cases/crowd-journalism/transactions')
count_data(res.json())
res_f = httpget(url = 'https://articonf1.itec.aau.at:30001/api/use_cases/crowd-journalism/transactions-failed')
count_data(res_f.json(), 'docType')
# failed tags: the "tag" is missing, but is called name
# failed purchases: duplicate keys generated from (userid, videoid, ownerid)
# failed classifications: impact is missing
# visualize content
import matplotlib.pyplot as plt
def visualize_video_coordinates():
geolocations = []
for entry in res.json():
if entry['table'] != 'video':
continue
loc_ = entry['properties']['geolocation'].split(',')
if loc_[0] == 'undefined' or loc_[1] == 'undefined':
continue
geolocations.append(loc_)
plt.scatter([float(coor[0]) for coor in geolocations], [float(coor[1]) for coor in geolocations])
plt.axis('off')
plt.show()
# visualize_video_coordinates()
def visualize_video_prices():
price = []
for entry in res.json():
if entry['table'] != 'video':
continue
price.append(entry['properties']['price'])
from collections import Counter
print(Counter(price))
plt.hist(price, bins=len(set(price)))
plt.show()
# visualize_video_prices()
def visualize_content_ratings():
impact = []
informative = []
trustiness = []
for entry in res.json():
if entry['table'] != 'classification':
continue
if entry['properties']['impact'] is not None:
impact.append(entry['properties']['impact'])
if entry['properties']['informative'] is not None:
informative.append(entry['properties']['informative'])
if entry['properties']['trustiness'] is not None:
trustiness.append(entry['properties']['trustiness'])
from collections import Counter
print(Counter(impact))
print(Counter(informative))
print(Counter(trustiness))
fig, (ax1, ax2, ax3) = plt.subplots(3)
ax1.hist(impact, bins=len(set(impact)))
ax1.set_title('impact')
ax2.hist(informative, bins=len(set(informative)))
ax2.set_title('informative')
ax3.hist(trustiness, bins=len(set(trustiness)))
ax3.set_title('trustiness')
plt.show()
# visualize_content_ratings()
# counting duplicate entries for 'purchase'
working_purchase_ids = [(entry['properties']['userid'], entry['properties']['videoid'], entry['properties']['ownerid']) for entry in res.json() if entry['table'] == 'purchase']
failed_purchase_ids = [(entry['userid'], entry['videoid'], entry['ownerid']) for entry in res_f.json() if entry['docType'] == 'purchase']
ic(len(working_purchase_ids))
ic(len(failed_purchase_ids))
cnt = 0
for failed_id in failed_purchase_ids:
if failed_id in working_purchase_ids:
cnt += 1
ic(cnt)
\ No newline at end of file
tools/check-use-case-data/reupload_failed_mog.py 0 → 100644
View file @ 0ba62690
import requests
requests.packages.urllib3.disable_warnings()
from icecream import ic
token = 'eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJ1c2VybmFtZSI6InJlZ3VsYXJAaXRlYy5hYXUuYXQiLCJjcmVhdGVkX2F0IjoiMjAyMS0wMy0yNCAxMDoxMzo1MS4wMjkwNDkiLCJ2YWxpZF91bnRpbCI6IjIwMjEtMDMtMjUgMTA6MTM6NTEuMDI5MDQ5In0.V6kYV5Lmb_tUIsF-6AKNB8_lIifmJP_Dm8gHhGa5w_o'
def httpget(url):
res = requests.get(url,
verify=False,
headers = { "Authorization": f"Bearer {token}"})
return res
res_f = httpget(url = 'https://articonf1.itec.aau.at:30001/api/use_cases/crowd-journalism/transactions-failed')
failed_purchases = []
for entry in res_f.json():
if entry['docType'] == 'purchase':
failed_purchases.append(entry)
print(len(failed_purchases))
# upload again
def httppost_gateway(content_):
url = 'https://articonf1.itec.aau.at:30401/api/trace'
res = requests.post(url,
verify=False,
headers = { "Authorization": f"Bearer {token}"},
json=content_)
return res
for purchase in failed_purchases:
res = httppost_gateway(purchase)
print(res)
\ No newline at end of file
tools/reddit-upload/upload_data.py
View file @ 0ba62690
...@@ -18,7 +18,7 @@ urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) ...@@ -18,7 +18,7 @@ urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
def send_transaction_to_rest_gateway(transaction: dict): def send_transaction_to_rest_gateway(transaction: dict):
# token from Rest Gateway to authorize # token from Rest Gateway to authorize
JWT_TOKEN = 'eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJ1c2VybmFtZSI6InJlZ3VsYXJAaXRlYy5hYXUuYXQiLCJjcmVhdGVkX2F0IjoiMjAyMS0wMi0wOCAxMzo0NzoxOC40NzUxMjEiLCJ2YWxpZF91bnRpbCI6IjIwMjEtMDItMDkgMTM6NDc6MTguNDc1MTIxIn0.DWY9c0X2XQJDz0Ef35-k1IVY6GWf00ogaVOCeX8Irlo' JWT_TOKEN = 'eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJ1c2VybmFtZSI6InJlZ3VsYXJAaXRlYy5hYXUuYXQiLCJjcmVhdGVkX2F0IjoiMjAyMS0wMy0xNiAxMzoxNDoyMS42MDc1NjciLCJ2YWxpZF91bnRpbCI6IjIwMjEtMDMtMTcgMTM6MTQ6MjEuNjA3NTY3In0.ZGObriEDWYo1BgiYN3pQSosS7UuNrq10GSCSjmRHSAw'
res = requests.post( res = requests.post(
url = 'https://articonf1.itec.aau.at:30401/api/trace', url = 'https://articonf1.itec.aau.at:30401/api/trace',
...@@ -47,7 +47,6 @@ if __name__ == '__main__': ...@@ -47,7 +47,6 @@ if __name__ == '__main__':
transaction['ApplicationType'] = 'reddit' transaction['ApplicationType'] = 'reddit'
transaction['docType'] = 'reddit' transaction['docType'] = 'reddit'
for key, value in obj_dict.items(): for key, value in obj_dict.items():
transaction[key] = value transaction[key] = value
...@@ -56,6 +55,7 @@ if __name__ == '__main__': ...@@ -56,6 +55,7 @@ if __name__ == '__main__':
if (summ % 1000 == 0 ): if (summ % 1000 == 0 ):
print ("Uploaded " + str(summ) + " transactions.") print ("Uploaded " + str(summ) + " transactions.")
if summ >= 1:
break
print ("TOTAL Uploaded " + str(summ) + " transactions.") print ("TOTAL Uploaded " + str(summ) + " transactions.")
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment