Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
S
SMART
Project
Project
Details
Activity
Releases
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
3
Issues
3
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Registry
Registry
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
UNI-KLU
SMART
Commits
643da815
Commit
643da815
authored
Mar 24, 2021
by
Alexander Lercher
Browse files
Options
Browse Files
Download
Plain Diff
Merge branch 'bugfix/duplicate-trace' into 'develop'
Bugfix/duplicate trace See merge request
!36
parents
5ed3c13e
6b8fd3ed
Changes
5
Show whitespace changes
Inline
Side-by-side
Showing
5 changed files
with
42 additions
and
33 deletions
+42
-33
repository.py
...n/trace-retrieval-microservice/app/database/repository.py
+6
-4
MessageHandler.py
...ce-retrieval-microservice/app/messaging/MessageHandler.py
+28
-23
transactions.py
...n/trace-retrieval-microservice/app/routes/transactions.py
+2
-1
test_MessageHandler.py
...e-retrieval-microservice/app/tests/test_MessageHandler.py
+5
-4
upload_data.py
tools/reddit-upload/upload_data.py
+1
-1
No files found.
src/transaction-hub-in/trace-retrieval-microservice/app/database/repository.py
View file @
643da815
...
@@ -31,7 +31,9 @@ class Repository(MongoRepositoryBase):
...
@@ -31,7 +31,9 @@ class Repository(MongoRepositoryBase):
@throws
@throws
KeyError - Duplicate transaction ID
KeyError - Duplicate transaction ID
'''
'''
reference
=
self
.
get_transaction_with_id
(
transaction
.
id
())
use_case
=
transaction
.
use_case
table
=
transaction
.
table
reference
=
self
.
get_transaction_with_id
(
transaction
.
id
(),
use_case
,
table
)
if
reference
==
None
:
if
reference
==
None
:
super
()
.
insert_entry
(
self
.
_transaction_collection
,
transaction
.
to_serializable_dict
())
super
()
.
insert_entry
(
self
.
_transaction_collection
,
transaction
.
to_serializable_dict
())
else
:
else
:
...
@@ -41,11 +43,11 @@ class Repository(MongoRepositoryBase):
...
@@ -41,11 +43,11 @@ class Repository(MongoRepositoryBase):
result
=
super
()
.
get_entries
(
self
.
_transaction_collection
,
projection
=
{
'_id'
:
False
},
selection
=
{
"use_case"
:
use_case
})
result
=
super
()
.
get_entries
(
self
.
_transaction_collection
,
projection
=
{
'_id'
:
False
},
selection
=
{
"use_case"
:
use_case
})
return
[
Transaction
.
from_serializable_dict
(
row
)
for
row
in
list
(
result
)]
return
[
Transaction
.
from_serializable_dict
(
row
)
for
row
in
list
(
result
)]
def
get_transaction_with_id
(
self
,
unique_id
:
str
)
->
Transaction
:
def
get_transaction_with_id
(
self
,
unique_id
:
str
,
use_case
:
str
,
table
:
str
)
->
Transaction
:
result
=
list
(
super
()
.
get_entries
(
self
.
_transaction_collection
,
projection
=
{
'_id'
:
False
},
selection
=
{
"
UniqueID"
:
unique_id
}))
result
=
list
(
super
()
.
get_entries
(
self
.
_transaction_collection
,
projection
=
{
'_id'
:
False
},
selection
=
{
"
id"
:
unique_id
,
"use_case"
:
use_case
,
"table"
:
table
}))
if
len
(
result
)
>=
1
:
if
len
(
result
)
>=
1
:
return
Transaction
.
from_serializable_dict
(
result
)
return
Transaction
.
from_serializable_dict
(
result
[
0
]
)
return
None
return
None
...
...
src/transaction-hub-in/trace-retrieval-microservice/app/messaging/MessageHandler.py
View file @
643da815
...
@@ -8,9 +8,11 @@ import json
...
@@ -8,9 +8,11 @@ import json
import
hashlib
import
hashlib
import
logging
import
logging
import
requests
import
requests
requests
.
packages
.
urllib3
.
disable_warnings
()
from
typing
import
Dict
from
typing
import
Dict
LOGGER
=
logging
.
getLogger
(
__name__
)
LOGGER
=
logging
.
getLogger
(
__name__
)
class
MessageHandler
:
class
MessageHandler
:
...
@@ -30,7 +32,7 @@ class MessageHandler:
...
@@ -30,7 +32,7 @@ class MessageHandler:
self
.
_rest_fetcher
=
rest_fetcher
self
.
_rest_fetcher
=
rest_fetcher
def
handle_generic
(
self
,
body
):
def
handle_generic
(
self
,
body
):
LOGGER
.
info
(
f
"Received message: {body}"
)
result
=
None
result
=
None
message
=
None
message
=
None
...
@@ -39,11 +41,13 @@ class MessageHandler:
...
@@ -39,11 +41,13 @@ class MessageHandler:
except
(
ValueError
,
TypeError
):
except
(
ValueError
,
TypeError
):
result
=
self
.
MSG_NOT_JSON
result
=
self
.
MSG_NOT_JSON
LOGGER
.
warning
(
result
)
LOGGER
.
warning
(
result
)
LOGGER
.
info
(
f
"Received message: {body}"
)
return
result
return
result
if
not
'type'
in
message
:
if
not
'type'
in
message
:
result
=
self
.
MSG_NO_TYPE
result
=
self
.
MSG_NO_TYPE
LOGGER
.
warning
(
result
)
LOGGER
.
warning
(
result
)
LOGGER
.
info
(
f
"Received message: {body}"
)
return
result
return
result
if
message
[
'type'
]
==
'blockchain-transaction'
:
if
message
[
'type'
]
==
'blockchain-transaction'
:
...
@@ -51,8 +55,11 @@ class MessageHandler:
...
@@ -51,8 +55,11 @@ class MessageHandler:
result
=
self
.
MSG_TRACE_PROCESSED
result
=
self
.
MSG_TRACE_PROCESSED
else
:
else
:
result
=
self
.
MSG_NOT_PROCESSED
result
=
self
.
MSG_NOT_PROCESSED
LOGGER
.
warning
(
result
)
LOGGER
.
info
(
f
"Received message: {body}"
)
LOGGER
.
info
(
result
)
#LOGGER.info(result) #too much spam
return
result
return
result
def
_resolve_path
(
self
,
data
:
Dict
,
path
:
str
)
->
Dict
:
def
_resolve_path
(
self
,
data
:
Dict
,
path
:
str
)
->
Dict
:
...
@@ -132,39 +139,39 @@ class MessageHandler:
...
@@ -132,39 +139,39 @@ class MessageHandler:
'''
'''
# check if there is a use-case in the message
# check if there is a use-case in the message
if
"ApplicationType"
not
in
transaction_message
.
keys
()
:
if
"ApplicationType"
not
in
transaction_message
:
LOGGER
.
error
(
"Transaction has no ApplicationType, storing it under use-case 'unknown'."
)
LOGGER
.
error
(
"Transaction has no ApplicationType, storing it under use-case 'unknown'."
)
transaction_message
[
"ApplicationType"
]
=
"unknown"
transaction_message
[
"ApplicationType"
]
=
"unknown"
self
.
_mongo_repo
.
add_failed_transaction
(
transaction_message
)
self
.
_mongo_repo
.
add_failed_transaction
(
transaction_message
)
return
return
# check if there is a
doctyp
e in the message
# check if there is a
tabl
e in the message
if
"docType"
not
in
transaction_message
.
keys
()
:
if
"docType"
not
in
transaction_message
:
LOGGER
.
error
(
"Transaction has no docType, storing it under
docTyp
e 'unknown'."
)
LOGGER
.
error
(
"Transaction has no docType, storing it under
tabl
e 'unknown'."
)
transaction_message
[
"docType"
]
=
"unknown"
transaction_message
[
"docType"
]
=
"unknown"
self
.
_mongo_repo
.
add_failed_transaction
(
transaction_message
)
self
.
_mongo_repo
.
add_failed_transaction
(
transaction_message
)
return
return
use_case
=
transaction_message
[
"ApplicationType"
]
transaction_
use_case
=
transaction_message
[
"ApplicationType"
]
docTyp
e
=
transaction_message
[
"docType"
]
transaction_tabl
e
=
transaction_message
[
"docType"
]
try
:
try
:
tables
=
self
.
_rest_fetcher
.
fetch_schema_information
(
use_case
)
tables
=
self
.
_rest_fetcher
.
fetch_schema_information
(
transaction_
use_case
)
except
ValueError
as
e
:
except
ValueError
as
e
:
LOGGER
.
error
(
f
"{e}
\n
Storing it as a failed transaction
."
)
LOGGER
.
error
(
f
"{e}
\n
Could not fetch schema, storing it as a failed transaction.
."
)
self
.
_mongo_repo
.
add_failed_transaction
(
transaction_message
)
self
.
_mongo_repo
.
add_failed_transaction
(
transaction_message
)
return
return
target_table
=
None
target_table
=
None
# find correct table
# find correct table
for
table
in
tables
:
for
table
in
tables
:
if
table
[
"name"
]
==
docTyp
e
:
if
table
[
"name"
]
==
transaction_tabl
e
:
target_table
=
table
target_table
=
table
break
break
# abort if table does not exist.
# abort if table does not exist.
if
target_table
==
None
:
if
target_table
==
None
:
LOGGER
.
error
(
f
"There is no table '{
docTyp
e}', storing it as a failed transaction."
)
LOGGER
.
error
(
f
"There is no table '{
transaction_tabl
e}', storing it as a failed transaction."
)
self
.
_mongo_repo
.
add_failed_transaction
(
transaction_message
)
self
.
_mongo_repo
.
add_failed_transaction
(
transaction_message
)
return
return
...
@@ -172,19 +179,17 @@ class MessageHandler:
...
@@ -172,19 +179,17 @@ class MessageHandler:
try
:
try
:
flattened
=
self
.
_flatten_transaction
(
transaction_message
,
mappings
)
flattened
=
self
.
_flatten_transaction
(
transaction_message
,
mappings
)
except
KeyError
as
e
:
except
KeyError
as
e
:
LOGGER
.
error
(
f
"Failed while flattening with KeyError: {str(e)}"
)
LOGGER
.
error
(
f
"Failed while flattening with KeyError: {str(e)}
, storing it as a failed transaction.
"
)
self
.
_mongo_repo
.
add_failed_transaction
(
transaction_message
)
self
.
_mongo_repo
.
add_failed_transaction
(
transaction_message
)
return
return
transaction
=
Transaction
(
use_case
,
target_table
[
"name"
],
flattened
)
transaction
=
Transaction
(
transaction_
use_case
,
target_table
[
"name"
],
flattened
)
#check for duplicates
#check for duplicates
try
:
try
:
references
=
self
.
_mongo_repo
.
get_transaction_with_id
(
transaction
.
id
())
references
=
self
.
_mongo_repo
.
get_transaction_with_id
(
transaction
.
id
()
,
transaction_use_case
,
transaction_table
)
if
references
!=
None
:
if
references
!=
None
:
for
item
in
references
:
LOGGER
.
info
(
"Found duplicate, storing it as a duplicated transaction."
)
if
(
item
.
table
==
transaction
.
table
)
and
(
item
.
use_case
==
transaction
.
use_case
):
LOGGER
.
error
(
"Found duplicate"
)
self
.
_mongo_repo
.
add_duplicated_transaction
(
transaction
)
self
.
_mongo_repo
.
add_duplicated_transaction
(
transaction
)
return
return
except
ValueError
as
e
:
except
ValueError
as
e
:
...
@@ -194,14 +199,14 @@ class MessageHandler:
...
@@ -194,14 +199,14 @@ class MessageHandler:
try
:
try
:
self
.
_mongo_repo
.
add_transaction
(
transaction
)
self
.
_mongo_repo
.
add_transaction
(
transaction
)
except
KeyError
as
e
:
except
KeyError
as
e
:
LOGGER
.
error
(
f
"{e}"
)
LOGGER
.
error
(
f
"{e}
, ignored {transaction_message}
"
)
self
.
_mongo_repo
.
add_failed_transaction
(
transaction_message
)
#
self._mongo_repo.add_failed_transaction(transaction_message)
return
return
msg
=
{
msg
=
{
"type"
:
"new-trace"
,
"type"
:
"new-trace"
,
"content"
:
transaction
.
to_serializable_dict
()
,
"content"
:
transaction
.
to_serializable_dict
()
}
}
msg_string
=
json
.
dumps
(
msg
)
msg_string
=
json
.
dumps
(
msg
)
...
...
src/transaction-hub-in/trace-retrieval-microservice/app/routes/transactions.py
View file @
643da815
...
@@ -23,7 +23,8 @@ def delete_all_failed_for_use_case(use_case: str):
...
@@ -23,7 +23,8 @@ def delete_all_failed_for_use_case(use_case: str):
return
Response
(
status
=
200
)
return
Response
(
status
=
200
)
def
all_duplicated_for_use_case
(
use_case
:
str
):
def
all_duplicated_for_use_case
(
use_case
:
str
):
return
_repository
.
all_duplicated_transactions_for_use_case
(
use_case
)
transactions
=
_repository
.
all_duplicated_transactions_for_use_case
(
use_case
)
return
[
t
.
to_serializable_dict
()
for
t
in
transactions
]
def
delete_all_duplicated_for_use_case
(
use_case
:
str
):
def
delete_all_duplicated_for_use_case
(
use_case
:
str
):
_repository
.
delete_all_duplicated_transactions
(
use_case
)
_repository
.
delete_all_duplicated_transactions
(
use_case
)
...
...
src/transaction-hub-in/trace-retrieval-microservice/app/tests/test_MessageHandler.py
View file @
643da815
...
@@ -18,14 +18,14 @@ class DummyMongoRepo:
...
@@ -18,14 +18,14 @@ class DummyMongoRepo:
def
add_transaction
(
self
,
transaction
):
def
add_transaction
(
self
,
transaction
):
self
.
added_transactions
.
append
(
transaction
)
self
.
added_transactions
.
append
(
transaction
)
def
get_transaction_with_id
(
self
,
unique_id
:
str
):
def
get_transaction_with_id
(
self
,
unique_id
:
str
,
use_case
,
table
):
result
=
[]
result
=
[]
for
trans
in
self
.
added_transactions
:
for
trans
in
self
.
added_transactions
:
transID
=
trans
.
id
()
transID
=
trans
.
id
()
if
transID
==
unique_id
:
if
transID
==
unique_id
and
trans
.
use_case
==
use_case
and
trans
.
table
==
table
:
result
.
append
(
trans
)
result
.
append
(
trans
)
if
len
(
result
)
>
0
:
if
len
(
result
)
>
0
:
return
result
return
result
[
0
]
return
None
return
None
...
@@ -198,7 +198,8 @@ class Test_MessageHandler(unittest.TestCase):
...
@@ -198,7 +198,8 @@ class Test_MessageHandler(unittest.TestCase):
self
.
handler
.
handle_blockchain_transaction
(
msg2
[
'content'
])
self
.
handler
.
handle_blockchain_transaction
(
msg2
[
'content'
])
self
.
assertEqual
(
len
(
self
.
repo
.
added_transactions
),
2
)
self
.
assertEqual
(
len
(
self
.
repo
.
added_transactions
),
2
)
def
test_handleBlockchainTransaction_multipleTransactions_3AddedUnique2Duplicate
(
self
):
def
test_handleBlockchainTransaction_multipleTransactions3SameIdDiffUseCaseTable_3AddedUnique2Duplicate
(
self
):
#print("Entered Test: 3Unique 2Dupli")
msg
=
self
.
_get_valid_message
()
msg
=
self
.
_get_valid_message
()
msg2
=
self
.
_get_valid_message2
()
msg2
=
self
.
_get_valid_message2
()
msg3
=
self
.
_get_valid_message3
()
msg3
=
self
.
_get_valid_message3
()
...
...
tools/reddit-upload/upload_data.py
View file @
643da815
...
@@ -57,7 +57,7 @@ if __name__ == '__main__':
...
@@ -57,7 +57,7 @@ if __name__ == '__main__':
"ApplicationType"
:
"debug"
,
"ApplicationType"
:
"debug"
,
"docType"
:
"pizza"
,
"docType"
:
"pizza"
,
"id"
:
1
,
"id"
:
1
,
"name"
:
"
MEXICAANA
"
,
"name"
:
"
Quatro Formagi
"
,
"dough"
:
{
"dough"
:
{
"type"
:
"wheat"
,
"type"
:
"wheat"
,
"spinach"
:
False
"spinach"
:
False
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment