Commit e7b081ad authored by Alfonso Orta's avatar Alfonso Orta

Merge branch 'develop' into 'staging'

Develop

See merge request !38
parents 108a70c8 5c630a95
......@@ -24,6 +24,7 @@ This token is used for authentication as _regular user_ on all microservices cur
```
{
"ApplicationType": "use-case identifier as string",
"docType": "use-case-table identifier as string",
"key": "value",
...
}
......@@ -91,4 +92,4 @@ Returns the computed similarity. Two clusters belonging to the SAME layer will b
## Connected Cluster
Intermediary data-structure used only by the function which computes the similarity. Clusters are connected only to other clusters belonging to a DIFFERENT layer.
```GET https://articonf1.itec.aau.at:30103/api/use_cases/{use_case}/tables{table}/connectedClusters``` returns all connected clusters for the given use-case and table.
```GET https://articonf1.itec.aau.at:30103/api/use_cases/{use_case}/tables/{table}/connectedClusters``` returns all connected clusters for the given use-case and table.
......@@ -18,9 +18,11 @@ from pathlib import Path
from env_info import is_running_locally, get_resources_path
from flask import request
from flask import redirect
from flask_cors import CORS
# load swagger config
app = connexion.App(__name__, specification_dir='configs/')
CORS(app.app)
@app.app.before_request
def before_request():
......
......@@ -14,6 +14,7 @@ cycler==0.10.0
decorator==4.4.1
Deprecated==1.2.7
Flask==1.1.1
Flask-Cors==3.0.10
idna==2.8
importlib-metadata==1.5.0
inflection==0.3.1
......
......@@ -20,6 +20,7 @@ from env_info import is_running_locally, get_resources_path
from messaging.ReconnectingMessageManager import ReconnectingMessageManager
from messaging.MessageHandler import MessageHandler
from flask import request
from flask_cors import CORS
from flask import redirect
# init message handler
......@@ -30,6 +31,7 @@ def message_received_callback(channel, method, properties, body):
# load swagger config
app = connexion.App(__name__, specification_dir='configs/')
CORS(app.app)
@app.app.before_request
def before_request():
......
......@@ -10,6 +10,7 @@ connexion==2.7.0
coverage==5.3.1
cryptography==3.1
Flask==1.1.2
Flask-Cors==3.0.10
idna==2.9
importlib-metadata==1.6.1
inflection==0.5.0
......
......@@ -9,7 +9,7 @@ for modules_path in modules_paths:
if os.path.exists(modules_path):
sys.path.insert(1, modules_path)
from _add_use_case_scripts.vialog.tables import add_user, add_video
from _add_use_case_scripts.vialog.tables import add_user, add_video, add_change
import network_constants as nc
from security.token_manager import TokenManager
......@@ -38,4 +38,4 @@ if __name__ == "__main__":
add_use_case(use_case)
add_user.main(use_case)
add_video.main(use_case)
\ No newline at end of file
add_change.main(use_case)
\ No newline at end of file
import sys
import os
from pathlib import Path
from typing import Dict, Any
import requests
modules_paths = ['.', '../../../modules/']
for modules_path in modules_paths:
if os.path.exists(modules_path):
sys.path.insert(1, modules_path)
from _add_use_case_scripts.vialog.tables import add_user, add_video, add_change
import network_constants as nc
from security.token_manager import TokenManager
def add_use_case(use_case: str):
#use_case = "vialog"
jwt = TokenManager.getInstance().getToken()
url = f"https://articonf1.itec.aau.at:30420/api/use-cases"
response = requests.post(
url,
verify=False,
proxies = { "http":None, "https":None },
headers = { "Authorization": f"Bearer {jwt}"},
json = {"name": use_case}
)
print(url+": "+str(response.content))
if __name__ == "__main__":
use_case = "vialog-new-enum"
# disable ssl warnings :)
requests.packages.urllib3.disable_warnings()
add_use_case(use_case)
add_video.main(use_case)
add_change.main(use_case)
\ No newline at end of file
from _add_use_case_scripts.requestPost import postLayersToSwagger, postTableToSwagger
def add_table(use_case: str, table_name: str):
'''
take the columns and add the mappings at the server
replace all "/"'s in the internal representation with a "_"
'''
columns = [
"changeType",
"changedValue",
"previousValue",
"newValue",
]
columns = { c : c for c in columns }
columns["UniqueID"] = "videoId+changeId"
columns["changeType"] = "enum(changeType)"
columns["changedValue"] = "enum(changedValue)"
table = {
"name": table_name,
"mappings": columns
}
postTableToSwagger(use_case,table)
def add_layers(use_case:str, table_name: str):
layers = [
{
"use_case": use_case,
"table": table_name,
"name": "Changetype_Layer",
"properties": [
"changeType",
],
"cluster_properties": [
"changeType",
]
},
{
"use_case": use_case,
"table": table_name,
"name": "Changedvalue_Layer",
"properties": [
"changedValue",
"previousValue",
"newValue"
],
"cluster_properties": [
"changedValue"
]
}
]
postLayersToSwagger(use_case,layers)
def main(use_case: str):
print("user")
table_name = "change"
add_table(use_case,table_name)
add_layers(use_case,table_name)
\ No newline at end of file
......@@ -6,27 +6,32 @@ def add_table(use_case: str, table_name: str):
'''
columns = [
# "docType",
"videoId",
"Video_Token",
"replyTo",
"Created",
"Duration",
"videoResolution",
"Label",
"ThreadId",
"Position",
"ModifiedDate",
"Views",
"ModeratedBy",
"CommunityManagerNotes",
"Rewards",
"Video_State",
"Video_Type"
'videoId',
'objType',
'eventName',
'video_token',
'replyTo',
'created',
'duration',
'videoResolution',
'label',
'threadId',
'position',
'views',
'moderatedBy',
'moderationDate',
'communityManagerNotes',
'rewards',
'video_state',
'video_type'
]
columns = { c : c for c in columns }
columns["UniqueID"] = "videoId"
columns["moderatedBy"] = "enum(moderatedBy)"
columns["video_state"] = "enum(video_state)"
columns["video_type"] = "enum(video_type)"
columns["videoResolution"] = "enum(videoResolution)"
table = {
"name": table_name,
......@@ -43,14 +48,14 @@ def add_layers(use_case:str, table_name: str):
"name": "Manager_Layer",
"properties": [
"UniqueID",
"ModifiedDate",
"ModeratedBy",
"Video_State",
"Video_Type"
"moderationDate",
"moderatedBy",
"video_state",
"video_type"
],
"cluster_properties": [
"ModeratedBy",
"Video_State"
"moderatedBy",
"video_state"
]
},
{
......@@ -59,16 +64,16 @@ def add_layers(use_case:str, table_name: str):
"name": "Video_Popularity_Layer",
"properties": [
"UniqueID",
"Label",
"Created",
"Views",
"Rewards",
"Video_State",
"Video_Type"
"label",
"created",
"views",
"rewards",
"video_state",
"video_type"
],
"cluster_properties": [
"Views",
"Video_Type"
"views",
"video_type"
]
},
{
......@@ -77,15 +82,15 @@ def add_layers(use_case:str, table_name: str):
"name": "Video_Age_Layer",
"properties": [
"UniqueID",
"Label",
"Created",
"Views",
"Rewards",
"Video_State",
"Video_Type"
"label",
"created",
"views",
"rewards",
"video_state",
"video_type"
],
"cluster_properties": [
"Created"
"created"
]
},
{
......@@ -94,15 +99,15 @@ def add_layers(use_case:str, table_name: str):
"name": "Rewards_Layer",
"properties": [
"UniqueID",
"Label",
"Created",
"Views",
"Rewards",
"Video_State",
"Video_Type"
"label",
"created",
"views",
"rewards",
"video_state",
"video_type"
],
"cluster_properties": [
"Rewards"
"rewards"
]
},
{
......@@ -111,14 +116,14 @@ def add_layers(use_case:str, table_name: str):
"name": "Video_Lenght_Layer",
"properties": [
"UniqueID",
"Created",
"Views",
"Duration",
"Video_State",
"Video_Type"
"created",
"views",
"duration",
"video_state",
"video_type"
],
"cluster_properties": [
"Duration"
"duration"
]
},
{
......@@ -127,11 +132,11 @@ def add_layers(use_case:str, table_name: str):
"name": "Video_Resolution_Layer",
"properties": [
"UniqueID",
"Created",
"Views",
"created",
"views",
"videoResolution",
"Video_State",
"Video_Type"
"video_state",
"video_type"
],
"cluster_properties": [
"videoResolution"
......
......@@ -21,12 +21,14 @@ import connexion
from security import swagger_util
from env_info import is_running_locally, get_resources_path
from flask import request
from flask_cors import CORS
from flask import redirect
# start app
if __name__ == '__main__':
app = connexion.App(__name__, specification_dir='configs/')
CORS(app.app)
@app.app.before_request
def before_request():
......
import requests
import json
videoListUrl = "https://dev758755.vialog.app/Videos/Meta/ListAll"
videoUrl = "https://dev758755.vialog.app/stat/events?type=video&id="
# token from Rest Gateway to authorize
JWT_TOKEN = 'eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJ1c2VybmFtZSI6InJlZ3VsYXJAaXRlYy5hYXUuYXQiLCJjcmVhdGVkX2F0IjoiMjAyMS0wNC0wNyAxMjo0OTo0MS43MTkzNjQiLCJ2YWxpZF91bnRpbCI6IjIwMjEtMDQtMDggMTI6NDk6NDEuNzE5MzY0In0.FN6qqBQeJSmXtS0-0dBiL-ojz6Ou7E5Tc9macrrhM4A'
useCase = "vialog-new-enum"
def send_transaction_to_rest_gateway(transaction: dict):
res = requests.post(
url = 'https://articonf1.itec.aau.at:30401/api/trace',
json = transaction,
headers = {"Authorization": f"Bearer {JWT_TOKEN}"},
verify = False # ignore ssl error
)
print(res)
videosRequest = requests.get(videoListUrl)
empty = set()
html = set()
if (videosRequest.status_code != 200):
print(f"Status: {videosRequest.status_code}")
dataCount = 0
for video in videosRequest.json():
dataCount += 1
id = video["videoId"]
videoRequest = requests.get(f"{videoUrl}{id}")
if videoRequest.status_code != 200:
print(f"Status: {videoRequest.status_code}")
if videoRequest.text != "" and not videoRequest.text.startswith("<!DOCTYPE html>"):
print(f"\n\n{videoUrl}{id}")
historyList = sorted(videoRequest.json()[0]["History"], key=lambda k: k['moderationDate'], reverse=True)
historyList.append(empty)
i = 0
changeList = []
for eventMap in historyList:
if historyList[i+1] == empty:
break
if i == 0:
lastState = eventMap
lastState["ApplicationType"] = useCase
lastState["docType"] = "video"
print(f"{lastState}\n\n\n")
send_transaction_to_rest_gateway(lastState)
if eventMap["eventName"] == 'r1eabcbdc8f5378b2ba71a1b6fe2038b Created' or eventMap["eventName"] == 'Created':
change = {"changeType": "Created", "changedValue": "video_state", "previousValue": "", "newValue":"Created"}
elif eventMap["eventName"] == "Restore":
change = {"changeType": "Restore", "changedValue": "", "previousValue": "", "newValue":""}
elif eventMap["eventName"] == "PositionChange":
change = {"changeType": "PositionChange", "changedValue": "position", "previousValue": historyList[i+1]["position"], "newValue": eventMap["position"]}
elif eventMap["eventName"] == "Hide":
change = {"changeType": "Hide", "changedValue": "video_state", "previousValue": historyList[i+1]["video_state"], "newValue": eventMap["video_state"]}
elif eventMap["eventName"] == "Publish":
change = {"changeType": "Publish", "changedValue": "video_state", "previousValue": historyList[i+1]["video_state"], "newValue":eventMap["video_state"]}
elif eventMap["eventName"] == "CMNote":
change = {"changeType": "CMNote", "changedValue": "communityManagerNotes", "previousValue": historyList[i+1]["communityManagerNotes"], "newValue":eventMap["communityManagerNotes"]}
elif eventMap["eventName"] == "Move":
change = {"changeType": "Move", "changedValue": "position", "previousValue": historyList[i+1]["position"], "newValue":eventMap["position"]}
elif eventMap["eventName"] == "VideoType":
change = {"changeType": "VideoType", "changedValue": "video_type", "previousValue": historyList[i+1]["video_type"], "newValue":eventMap["video_type"]}
elif eventMap["eventName"] == "Delete":
change = {"changeType": "Delete", "changedValue": "", "previousValue": "", "newValue": ""}
elif eventMap["eventName"] == "Copy":
change = {"changeType": "Copy", "changedValue": "", "previousValue": "", "newValue":""}
elif eventMap["eventName"] == "CustomLabel":
change = {"changeType": "CustomLabel", "changedValue": "label", "previousValue": historyList[i+1]["label"], "newValue":eventMap["label"]}
change["videoId"] = id
change["changeId"] = i
change["timestamp"] = eventMap["moderationDate"]
change["ApplicationType"] = useCase
change["docType"] = "change"
print(change)
send_transaction_to_rest_gateway(change)
i += 1
elif videoRequest.text == "":
empty.add(id)
else:
html.add(id)
print(f"empty: {empty}\n\n")
print(f"html page: {html}\n\n")
print(f"history: {historyList}")
print(dataCount)
......@@ -9,10 +9,10 @@ click==7.1.2
clickclick==1.2.2
colorama==0.4.3
connexion==2.7.0
coverage==5.3.1
cryptography==2.9.2
Deprecated==1.2.10
Flask==1.1.2
Flask-Cors==3.0.10
idna==2.10
importlib-metadata==1.7.0
inflection==0.5.0
......
......@@ -31,7 +31,9 @@ class Repository(MongoRepositoryBase):
@throws
KeyError - Duplicate transaction ID
'''
reference = self.get_transaction_with_id(transaction.id())
use_case= transaction.use_case
table = transaction.table
reference = self.get_transaction_with_id(transaction.id(),use_case, table)
if reference == None:
super().insert_entry(self._transaction_collection, transaction.to_serializable_dict())
else:
......@@ -41,10 +43,10 @@ class Repository(MongoRepositoryBase):
result = super().get_entries(self._transaction_collection, projection={'_id': False}, selection={"use_case": use_case})
return [Transaction.from_serializable_dict(row) for row in list(result)]
def get_transaction_with_id(self, unique_id: str) -> Transaction:
result = list(super().get_entries(self._transaction_collection, projection={'_id': False}, selection={"UniqueID": unique_id}))
def get_transaction_with_id(self, unique_id: str, use_case:str, table:str ) -> Transaction:
result = list(super().get_entries(self._transaction_collection, projection={'_id': False}, selection={"id": unique_id,"use_case": use_case, "table":table}))
if len(result) == 1:
if len(result) >= 1:
return Transaction.from_serializable_dict(result[0])
return None
......
......@@ -8,9 +8,11 @@ import json
import hashlib
import logging
import requests
requests.packages.urllib3.disable_warnings()
from typing import Dict
LOGGER = logging.getLogger(__name__)
class MessageHandler:
......@@ -30,7 +32,7 @@ class MessageHandler:
self._rest_fetcher = rest_fetcher
def handle_generic(self, body):
LOGGER.info(f"Received message: {body}")
result = None
message = None
......@@ -39,11 +41,13 @@ class MessageHandler:
except (ValueError, TypeError):
result = self.MSG_NOT_JSON
LOGGER.warning(result)
LOGGER.info(f"Received message: {body}")
return result
if not 'type' in message:
result = self.MSG_NO_TYPE
LOGGER.warning(result)
LOGGER.info(f"Received message: {body}")
return result
if message['type'] == 'blockchain-transaction':
......@@ -51,8 +55,11 @@ class MessageHandler:
result = self.MSG_TRACE_PROCESSED
else:
result = self.MSG_NOT_PROCESSED
LOGGER.warning(result)
LOGGER.info(f"Received message: {body}")
LOGGER.info(result)
#LOGGER.info(result) #too much spam
return result
def _resolve_path(self, data: Dict, path:str) -> Dict:
......@@ -132,39 +139,39 @@ class MessageHandler:
'''
# check if there is a use-case in the message
if "ApplicationType" not in transaction_message.keys():
if "ApplicationType" not in transaction_message:
LOGGER.error("Transaction has no ApplicationType, storing it under use-case 'unknown'.")
transaction_message["ApplicationType"] = "unknown"
self._mongo_repo.add_failed_transaction(transaction_message)
return
# check if there is a doctype in the message
if "docType" not in transaction_message.keys():
LOGGER.error("Transaction has no docType, storing it under docType 'unknown'.")
# check if there is a table in the message
if "docType" not in transaction_message:
LOGGER.error("Transaction has no docType, storing it under table 'unknown'.")
transaction_message["docType"] = "unknown"
self._mongo_repo.add_failed_transaction(transaction_message)
return
use_case = transaction_message["ApplicationType"]
docType = transaction_message["docType"]
transaction_use_case = transaction_message["ApplicationType"]
transaction_table = transaction_message["docType"]
try:
tables = self._rest_fetcher.fetch_schema_information(use_case)
tables = self._rest_fetcher.fetch_schema_information(transaction_use_case)
except ValueError as e:
LOGGER.error(f"{e}\nStoring it as a failed transaction.")
LOGGER.error(f"{e}\nCould not fetch schema, storing it as a failed transaction..")
self._mongo_repo.add_failed_transaction(transaction_message)
return
target_table = None
# find correct table
for table in tables:
if table["name"] == docType:
if table["name"] == transaction_table:
target_table = table
break
# abort if table does not exist.
if target_table == None:
LOGGER.error(f"There is no table '{docType}', storing it as a failed transaction.")
LOGGER.error(f"There is no table '{transaction_table}', storing it as a failed transaction.")
self._mongo_repo.add_failed_transaction(transaction_message)
return
......@@ -172,20 +179,19 @@ class MessageHandler:
try:
flattened = self._flatten_transaction(transaction_message, mappings)
except KeyError as e:
LOGGER.error(f"Failed while flattening with KeyError: {str(e)}")
LOGGER.error(f"Failed while flattening with KeyError: {str(e)}, storing it as a failed transaction.")
self._mongo_repo.add_failed_transaction(transaction_message)
return
transaction = Transaction(use_case, target_table["name"], flattened)
transaction = Transaction(transaction_use_case, target_table["name"], flattened)
#check for duplicates
try:
reference = self._mongo_repo.get_transaction_with_id(transaction.id())
if reference != None:
if (reference[0].table == transaction.table) and (reference[0].use_case == transaction.use_case):
LOGGER.error("Found duplicate")
self._mongo_repo.add_duplicated_transaction(transaction)
return
references = self._mongo_repo.get_transaction_with_id(transaction.id(),transaction_use_case,transaction_table)
if references != None:
LOGGER.info("Found duplicate, storing it as a duplicated transaction.")
self._mongo_repo.add_duplicated_transaction(transaction)
return
except ValueError as e:
LOGGER.error(f"{e}, could not insert duplicated node.")
return
......@@ -193,14 +199,14 @@ class MessageHandler:
try:
self._mongo_repo.add_transaction(transaction)
except KeyError as e:
LOGGER.error(f"{e}")
self._mongo_repo.add_failed_transaction(transaction_message)
LOGGER.error(f"{e}, ignored {transaction_message}")
# self._mongo_repo.add_failed_transaction(transaction_message)
return
msg = {
"type": "new-trace",
"content": transaction.to_serializable_dict(),
"content": transaction.to_serializable_dict()
}
msg_string = json.dumps(msg)
......
......@@ -23,7 +23,8 @@ def delete_all_failed_for_use_case(use_case: str):
return Response(status=200)
def all_duplicated_for_use_case(use_case: str):
return _repository.all_duplicated_transactions_for_use_case(use_case)
transactions = _repository.all_duplicated_transactions_for_use_case(use_case)
return [t.to_serializable_dict() for t in transactions]
def delete_all_duplicated_for_use_case(use_case: str):
_repository.delete_all_duplicated_transactions(use_case)
......