Finished Duplicate Check and UnitTests for traces

src/data-hub/semantic-linking-microservice/app/db/repository.py

@@ -97,13 +97,14 @@ class Repository(MongoRepositoryBase):
                                       projection={'_id': 0})
         return list(entries)
 
-    def get_nodes_from_ids(self, unique_id_list: List) -> List[str]:
-        result = list(super().get_entries(self._transaction_collection, projection={'_id': False}, selection={"UniqueID": { "$in": unique_id_list}}))
+    # #TODO redundant?
+    # def get_nodes_from_ids(self, unique_id_list: List) -> List[str]:
+    #     result = list(super().get_entries(self._transaction_collection, projection={'_id': False}, selection={"UniqueID": { "$in": unique_id_list}}))
 
-        if len(result) > 0:
-            return result
+    #     if len(result) > 0:
+    #         return result
         
-        return None
+    #     return []
 
     # endregion
 

src/data-hub/semantic-linking-microservice/app/messaging/MessageHandler.py

@@ -97,8 +97,7 @@ class MessageHandler:
             nodes.append(node)
             unique_id_list.append(node["UniqueID"])
         
-        #check for duplicates
-        #TODO EDIT NODES_IN_DB RETURN
+        #TODO redundant if we already check in trace?
         # nodes_in_db = self._repository.get_nodes_from_ids(unique_id_list)
         # if len(nodes_in_db) > 0: #found duplicates
         #     #remove duplicates from nodes:
@@ -106,7 +105,6 @@ class MessageHandler:
         #         if node["UniqueID"] in nodes_in_db:
         #             nodes.remove(node)
 
-        #no duplicates anymore
         if len(nodes) > 0:
             self._repository.add_layer_nodes(nodes)
 

src/data-hub/semantic-linking-microservice/app/tests/test_pipeline.py

@@ -11,7 +11,7 @@ from typing import List
 class DummyMongoRepo:
     '''Dummy class to be used for testing the MessageHandler'''
     last_trace = None
-    DBnodes = []
+    layernodes = []
     def insert_trace(self, trace):
         self.last_trace = trace
     
@@ -35,53 +35,13 @@ class DummyMongoRepo:
         ]
 
     def add_layer_nodes(self, nodes: List):
-        pass
+        self.layernodes.extend(nodes)
+        return
 
-    def get_layers_for_use_case_and_table(use_case, table):
+    def get_nodes_from_ids(self, unique_id_list)
+        retList = []
+        for node in layernodes:
 
-        testLayers = [
-            {
-                "layer_name": "layer1",
-                "properties": [
-                                "UniqueID",
-                                "doughType"                                
-                                ],
-                "table": "pizza",
-                "total_properties": [
-                                "UniqueID",
-                                "doughType",
-                                "firstTopping",
-                                "firstToppingPrice",
-                                "hasSpinach",
-                                "name",
-                                "toppingInfo"
-                                ],
-                "use_case": "debug"
-            },
-            {
-                "layer_name": "layer2",
-                "properties": [
-                                "firstTopping"                            
-                                ],
-                "table": "pizza",
-                "total_properties": [
-                                "UniqueID",
-                                "doughType",
-                                "firstTopping",
-                                "firstToppingPrice",
-                                "hasSpinach",
-                                "name",
-                                "toppingInfo"
-                                ],
-                "use_case": "debug"
-            }
-        ]
-        returnLayers = []
-        for layer in testLayers:
-            if (layer["use_case"] == use_case) and (layer["table"] == table):
-                returnLayers.append(layer)
-
-        return returnLayers
 
     # layers = self._repository.get_layers_for_use_case_and_table(use_case, table)
 
@@ -95,11 +55,6 @@ class DummyMongoRepo:
     # nodes_in_db = self._repository.get_nodes_from_ids(unique_id_list)
     
 
-    #TODO first
-    # self._repository.add_layer_nodes(nodes)
-    def add_layer_nodessss(nodes):
-        DBnodes.extend(nodes)
-        return
         
 
 
@@ -135,14 +90,8 @@ class Test_Pipeline(unittest.TestCase):
     def testTraceProcessing(self):
         msg = self._buildTraceMessage()
         self.handler.handle_new_trace(msg["content"])
+        self.assertEqual(len(self.handler._repository.layernodes),1)
     
-    def testDuplicateTrace(self):
-        msg = self._buildTraceMessage()
-        msg2 = self._buildTraceMessage()
-        self.handler.handle_new_trace(msg["content"])
-        self.handler.handle_new_trace(msg2["content"])
-
-        self.assertEqual(len(self.repo.DBnodes),1)
 
 if __name__ == '__main__':
     unittest.main()
\ No newline at end of file

src/transaction-hub-in/trace-retrieval-microservice/app/messaging/MessageHandler.py

@@ -179,6 +179,7 @@ class MessageHandler:
         try:
             reference = self._mongo_repo.get_transaction_with_id(transaction.id())
             if reference != None:
+                if (reference[0].table == transaction.table) and (reference[0].use_case == transaction.use_case):
                     LOGGER.error("Found duplicate")
                     self._mongo_repo.add_duplicated_transaction(transaction)
                     return 

src/transaction-hub-in/trace-retrieval-microservice/app/messaging/dummy_rest_fetcher.py

@@ -2,7 +2,10 @@ from messaging.rest_fetcher import RestFetcher
 
 class DummyRestFetcher(RestFetcher):
     def fetch_schema_information(self, use_case: str):
-        return [
+        returnList = []
+        
+        if use_case == "string":
+            returnList =[
                         {
                             "name": "string",
                             "use_case": "string",
@@ -10,5 +13,33 @@ class DummyRestFetcher(RestFetcher):
                                 "UniqueID": "ResourceIds",
                                 "RIds": "ResourceIds"
                             }
+                        },
+                        {
+                            "name": "string2",
+                            "use_case": "string",
+                            "mappings": {
+                                "UniqueID": "ResourceIds",
+                                "RIds": "ResourceIds"
+                            }
+                        }
+                    ]
+        else:
+            returnList = [
+                        {
+                            "name": "string",
+                            "use_case": "string2",
+                            "mappings": {
+                                "UniqueID": "ResourceIds",
+                                "RIds": "ResourceIds"
+                            }
+                        },
+                        {
+                            "name": "string2",
+                            "use_case": "string2",
+                            "mappings": {
+                                "UniqueID": "ResourceIds",
+                                "RIds": "ResourceIds"
+                            }
                         }
                     ]
+        return returnList
\ No newline at end of file

src/transaction-hub-in/trace-retrieval-microservice/app/tests/test_MessageHandler.py

@@ -54,7 +54,7 @@ class Test_MessageHandler(unittest.TestCase):
         { 'type': 'blockchain-transaction',
           'content': 
             {
-            "ApplicationType": "paper",
+            "ApplicationType": "string",
             "docType": "string",
             "Metadata": {},
             "ResourceIds": "string",
@@ -70,6 +70,48 @@ class Test_MessageHandler(unittest.TestCase):
             }
         }
         return json.dumps(message_values)
+    def _get_valid_message2(self) -> str:
+        message_values = \
+        { 'type': 'blockchain-transaction',
+          'content': 
+            {
+            "ApplicationType": "string2",
+            "docType": "string",
+            "Metadata": {},
+            "ResourceIds": "string",
+            "ResourceMd5": "string",
+            "ResourceState": "string",
+            "Timestamp": "2019-08-27T14:00:48.587Z",
+            "TransactionFrom": "string",
+            "TransactionFromLatLng": "string",
+            "TransactionId": "string",
+            "TransactionTo": "string",
+            "TransactionToLatLng": "string",
+            "TransferredAsset": "string"
+            }
+        }
+        return json.dumps(message_values)
+    def _get_valid_message3(self) -> str:
+        message_values = \
+        { 'type': 'blockchain-transaction',
+          'content': 
+            {
+            "ApplicationType": "string",
+            "docType": "string2",
+            "Metadata": {},
+            "ResourceIds": "string",
+            "ResourceMd5": "string",
+            "ResourceState": "string",
+            "Timestamp": "2019-08-27T14:00:48.587Z",
+            "TransactionFrom": "string",
+            "TransactionFromLatLng": "string",
+            "TransactionId": "string",
+            "TransactionTo": "string",
+            "TransactionToLatLng": "string",
+            "TransferredAsset": "string"
+            }
+        }
+        return json.dumps(message_values)
 
     def test_handleGeneric_emptyMessage_NotJsonError(self):
         res = self.handler.handle_generic('')
@@ -137,5 +179,23 @@ class Test_MessageHandler(unittest.TestCase):
         self.handler.handle_blockchain_transaction(msg2['content'])
         self.assertEqual(len(self.repo.added_transactions),len(self.repo.duplicated_transactions))
 
+    def test_handleblockchain_duplicateTraceDifferentTable(self):
+        msg = self._get_valid_message()
+        msg2 = self._get_valid_message2()
+        msg = eval(msg)
+        msg2 = eval(msg2)
+        self.handler.handle_blockchain_transaction(msg['content'])
+        self.handler.handle_blockchain_transaction(msg2['content'])
+        self.assertEqual(len(self.repo.added_transactions),2)
+
+    def test_handleblockchain_duplicateTraceDifferentUseCase(self):
+        msg = self._get_valid_message()
+        msg2 = self._get_valid_message3()
+        msg = eval(msg)
+        msg2 = eval(msg2)
+        self.handler.handle_blockchain_transaction(msg['content'])
+        self.handler.handle_blockchain_transaction(msg2['content'])
+        self.assertEqual(len(self.repo.added_transactions),2)
+
 if __name__ == '__main__':
     unittest.main()
\ No newline at end of file