Skip to content

S
SMART
Commit 31679a46 authored by Bogdan's avatar Bogdan
Browse files
Options

Finished Duplicate Check and UnitTests for traces

parent e84e36cf
Hide whitespace changes
Inline Side-by-side
Showing with 121 additions and 81 deletions
src/data-hub/semantic-linking-microservice/app/db/repository.py
View file @ 31679a46
...@@ -97,13 +97,14 @@ class Repository(MongoRepositoryBase): ...@@ -97,13 +97,14 @@ class Repository(MongoRepositoryBase):
projection={'_id': 0}) projection={'_id': 0})
return list(entries) return list(entries)
def get_nodes_from_ids(self, unique_id_list: List) -> List[str]: # #TODO redundant?
result = list(super().get_entries(self._transaction_collection, projection={'_id': False}, selection={"UniqueID": { "$in": unique_id_list}})) # def get_nodes_from_ids(self, unique_id_list: List) -> List[str]:
# result = list(super().get_entries(self._transaction_collection, projection={'_id': False}, selection={"UniqueID": { "$in": unique_id_list}}))
if len(result) > 0: # if len(result) > 0:
return result # return result
return None # return []
# endregion # endregion
src/data-hub/semantic-linking-microservice/app/messaging/MessageHandler.py
View file @ 31679a46
...@@ -97,8 +97,7 @@ class MessageHandler: ...@@ -97,8 +97,7 @@ class MessageHandler:
nodes.append(node) nodes.append(node)
unique_id_list.append(node["UniqueID"]) unique_id_list.append(node["UniqueID"])
#check for duplicates #TODO redundant if we already check in trace?
#TODO EDIT NODES_IN_DB RETURN
# nodes_in_db = self._repository.get_nodes_from_ids(unique_id_list) # nodes_in_db = self._repository.get_nodes_from_ids(unique_id_list)
# if len(nodes_in_db) > 0: #found duplicates # if len(nodes_in_db) > 0: #found duplicates
# #remove duplicates from nodes: # #remove duplicates from nodes:
...@@ -106,7 +105,6 @@ class MessageHandler: ...@@ -106,7 +105,6 @@ class MessageHandler:
# if node["UniqueID"] in nodes_in_db: # if node["UniqueID"] in nodes_in_db:
# nodes.remove(node) # nodes.remove(node)
#no duplicates anymore
if len(nodes) > 0: if len(nodes) > 0:
self._repository.add_layer_nodes(nodes) self._repository.add_layer_nodes(nodes)
......
src/data-hub/semantic-linking-microservice/app/tests/test_pipeline.py
View file @ 31679a46
...@@ -11,7 +11,7 @@ from typing import List ...@@ -11,7 +11,7 @@ from typing import List
class DummyMongoRepo: class DummyMongoRepo:
'''Dummy class to be used for testing the MessageHandler''' '''Dummy class to be used for testing the MessageHandler'''
last_trace = None last_trace = None
DBnodes = [] layernodes = []
def insert_trace(self, trace): def insert_trace(self, trace):
self.last_trace = trace self.last_trace = trace
...@@ -35,53 +35,13 @@ class DummyMongoRepo: ...@@ -35,53 +35,13 @@ class DummyMongoRepo:
] ]
def add_layer_nodes(self, nodes: List): def add_layer_nodes(self, nodes: List):
pass self.layernodes.extend(nodes)
return
def get_layers_for_use_case_and_table(use_case, table):
testLayers = [ def get_nodes_from_ids(self, unique_id_list)
{ retList = []
"layer_name": "layer1", for node in layernodes:
"properties": [
"UniqueID",
"doughType"
],
"table": "pizza",
"total_properties": [
"UniqueID",
"doughType",
"firstTopping",
"firstToppingPrice",
"hasSpinach",
"name",
"toppingInfo"
],
"use_case": "debug"
},
{
"layer_name": "layer2",
"properties": [
"firstTopping"
],
"table": "pizza",
"total_properties": [
"UniqueID",
"doughType",
"firstTopping",
"firstToppingPrice",
"hasSpinach",
"name",
"toppingInfo"
],
"use_case": "debug"
}
]
returnLayers = []
for layer in testLayers:
if (layer["use_case"] == use_case) and (layer["table"] == table):
returnLayers.append(layer)
return returnLayers
# layers = self._repository.get_layers_for_use_case_and_table(use_case, table) # layers = self._repository.get_layers_for_use_case_and_table(use_case, table)
...@@ -94,12 +54,7 @@ class DummyMongoRepo: ...@@ -94,12 +54,7 @@ class DummyMongoRepo:
#TODO #TODO
# nodes_in_db = self._repository.get_nodes_from_ids(unique_id_list) # nodes_in_db = self._repository.get_nodes_from_ids(unique_id_list)
#TODO first
# self._repository.add_layer_nodes(nodes)
def add_layer_nodessss(nodes):
DBnodes.extend(nodes)
return
...@@ -135,14 +90,8 @@ class Test_Pipeline(unittest.TestCase): ...@@ -135,14 +90,8 @@ class Test_Pipeline(unittest.TestCase):
def testTraceProcessing(self): def testTraceProcessing(self):
msg = self._buildTraceMessage() msg = self._buildTraceMessage()
self.handler.handle_new_trace(msg["content"]) self.handler.handle_new_trace(msg["content"])
self.assertEqual(len(self.handler._repository.layernodes),1)
def testDuplicateTrace(self):
msg = self._buildTraceMessage()
msg2 = self._buildTraceMessage()
self.handler.handle_new_trace(msg["content"])
self.handler.handle_new_trace(msg2["content"])
self.assertEqual(len(self.repo.DBnodes),1)
if __name__ == '__main__': if __name__ == '__main__':
unittest.main() unittest.main()
\ No newline at end of file
src/transaction-hub-in/trace-retrieval-microservice/app/messaging/MessageHandler.py
View file @ 31679a46
...@@ -179,9 +179,10 @@ class MessageHandler: ...@@ -179,9 +179,10 @@ class MessageHandler:
try: try:
reference = self._mongo_repo.get_transaction_with_id(transaction.id()) reference = self._mongo_repo.get_transaction_with_id(transaction.id())
if reference != None: if reference != None:
LOGGER.error("Found duplicate") if (reference[0].table == transaction.table) and (reference[0].use_case == transaction.use_case):
self._mongo_repo.add_duplicated_transaction(transaction) LOGGER.error("Found duplicate")
return self._mongo_repo.add_duplicated_transaction(transaction)
return
except ValueError as e: except ValueError as e:
LOGGER.error(f"{e}, could not insert duplicated node.") LOGGER.error(f"{e}, could not insert duplicated node.")
return return
......
src/transaction-hub-in/trace-retrieval-microservice/app/messaging/dummy_rest_fetcher.py
View file @ 31679a46
...@@ -2,13 +2,44 @@ from messaging.rest_fetcher import RestFetcher ...@@ -2,13 +2,44 @@ from messaging.rest_fetcher import RestFetcher
class DummyRestFetcher(RestFetcher): class DummyRestFetcher(RestFetcher):
def fetch_schema_information(self, use_case: str): def fetch_schema_information(self, use_case: str):
return [ returnList = []
{
"name": "string", if use_case == "string":
"use_case": "string", returnList =[
"mappings": { {
"UniqueID": "ResourceIds", "name": "string",
"RIds": "ResourceIds" "use_case": "string",
} "mappings": {
} "UniqueID": "ResourceIds",
] "RIds": "ResourceIds"
}
},
{
"name": "string2",
"use_case": "string",
"mappings": {
"UniqueID": "ResourceIds",
"RIds": "ResourceIds"
}
}
]
else:
returnList = [
{
"name": "string",
"use_case": "string2",
"mappings": {
"UniqueID": "ResourceIds",
"RIds": "ResourceIds"
}
},
{
"name": "string2",
"use_case": "string2",
"mappings": {
"UniqueID": "ResourceIds",
"RIds": "ResourceIds"
}
}
]
return returnList
\ No newline at end of file
src/transaction-hub-in/trace-retrieval-microservice/app/tests/test_MessageHandler.py
View file @ 31679a46
...@@ -54,7 +54,7 @@ class Test_MessageHandler(unittest.TestCase): ...@@ -54,7 +54,7 @@ class Test_MessageHandler(unittest.TestCase):
{ 'type': 'blockchain-transaction', { 'type': 'blockchain-transaction',
'content': 'content':
{ {
"ApplicationType": "paper", "ApplicationType": "string",
"docType": "string", "docType": "string",
"Metadata": {}, "Metadata": {},
"ResourceIds": "string", "ResourceIds": "string",
...@@ -70,6 +70,48 @@ class Test_MessageHandler(unittest.TestCase): ...@@ -70,6 +70,48 @@ class Test_MessageHandler(unittest.TestCase):
} }
} }
return json.dumps(message_values) return json.dumps(message_values)
def _get_valid_message2(self) -> str:
message_values = \
{ 'type': 'blockchain-transaction',
'content':
{
"ApplicationType": "string2",
"docType": "string",
"Metadata": {},
"ResourceIds": "string",
"ResourceMd5": "string",
"ResourceState": "string",
"Timestamp": "2019-08-27T14:00:48.587Z",
"TransactionFrom": "string",
"TransactionFromLatLng": "string",
"TransactionId": "string",
"TransactionTo": "string",
"TransactionToLatLng": "string",
"TransferredAsset": "string"
}
}
return json.dumps(message_values)
def _get_valid_message3(self) -> str:
message_values = \
{ 'type': 'blockchain-transaction',
'content':
{
"ApplicationType": "string",
"docType": "string2",
"Metadata": {},
"ResourceIds": "string",
"ResourceMd5": "string",
"ResourceState": "string",
"Timestamp": "2019-08-27T14:00:48.587Z",
"TransactionFrom": "string",
"TransactionFromLatLng": "string",
"TransactionId": "string",
"TransactionTo": "string",
"TransactionToLatLng": "string",
"TransferredAsset": "string"
}
}
return json.dumps(message_values)
def test_handleGeneric_emptyMessage_NotJsonError(self): def test_handleGeneric_emptyMessage_NotJsonError(self):
res = self.handler.handle_generic('') res = self.handler.handle_generic('')
...@@ -137,5 +179,23 @@ class Test_MessageHandler(unittest.TestCase): ...@@ -137,5 +179,23 @@ class Test_MessageHandler(unittest.TestCase):
self.handler.handle_blockchain_transaction(msg2['content']) self.handler.handle_blockchain_transaction(msg2['content'])
self.assertEqual(len(self.repo.added_transactions),len(self.repo.duplicated_transactions)) self.assertEqual(len(self.repo.added_transactions),len(self.repo.duplicated_transactions))
def test_handleblockchain_duplicateTraceDifferentTable(self):
msg = self._get_valid_message()
msg2 = self._get_valid_message2()
msg = eval(msg)
msg2 = eval(msg2)
self.handler.handle_blockchain_transaction(msg['content'])
self.handler.handle_blockchain_transaction(msg2['content'])
self.assertEqual(len(self.repo.added_transactions),2)
def test_handleblockchain_duplicateTraceDifferentUseCase(self):
msg = self._get_valid_message()
msg2 = self._get_valid_message3()
msg = eval(msg)
msg2 = eval(msg2)
self.handler.handle_blockchain_transaction(msg['content'])
self.handler.handle_blockchain_transaction(msg2['content'])
self.assertEqual(len(self.repo.added_transactions),2)
if __name__ == '__main__': if __name__ == '__main__':
unittest.main() unittest.main()
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment