Skip to content

S
SMART
Commit 271733ea authored by Alexander Lercher's avatar Alexander Lercher
Browse files
Options

Merge branch 'develop' into feature/community-prediction

parents dc6fbe8e 87cf2ccd
Show whitespace changes
Inline Side-by-side
Showing with 3195 additions and 180 deletions
.gitignore
View file @ 271733ea
......@@ -17,3 +17,5 @@ src/modules/security/regular_user_credentials.json
src/modules/security/default_users.json
resources/
reports/
bin/testing.py
View file @ 271733ea
......@@ -2,7 +2,8 @@ import os
import sys
import importlib.util
import pathlib
import shutil
import re
'''
This script searches for all 'tests/' directories and executes all tests
by cd'ing into the dir and executing unittest discover.
......@@ -12,29 +13,107 @@ Use command line argument '-w' to run on windows.
PY = sys.argv[2] if (len(sys.argv) > 1 and sys.argv[1] == '-py') else 'python3.7' # use -py to use your own python command
ROOT = pathlib.Path(__file__).parent.parent.absolute()
REPORTS = ROOT / 'reports'
TESTS_FOLDER_NAME = os.path.normpath("/tests")
print("Creating VENV")
os.system(f"{PY} -m venv venv")
PY = f"~/smart/venv/bin/{PY}"
print("\nSearching for tests at the path: "+ str(ROOT))
count = 0
resultCodeList = []
microservice_coverage_paths_set = set()
for (dirname, dirs, files) in os.walk(ROOT):
#I assume all the tests are placed in a folder named "tests"
if (TESTS_FOLDER_NAME in str(dirname)) \
and 'src' in str(dirname) \
and not(f"{TESTS_FOLDER_NAME}{os.path.normpath('/')}" in str(dirname)) \
and not("venv" in str(dirname)):
and not("venv" in str(dirname)) \
and not("Lib" in str(dirname)):
try:
print(f"Executing tests in {dirname}")
os.chdir(os.path.normpath(dirname))
# TODO do this during docker image setup
exit_val = os.system(f"{PY} -m pip install -r ../requirements.txt") # install pip dependencies
exit_val = os.system(f"{PY} -m unittest discover") # execute the tests
#resultCodeList.append(exit_val)
#exit_val = os.system(f"{PY} -m unittest discover") # execute the tests
exit_val = os.system(f"{PY} -m coverage run --append --omit=*/site-packages*,*/dist-packages* -m unittest discover") #TEST CODE COVERAGE
microservice_coverage_paths_set.add(os.path.normpath(dirname))
resultCodeList.append(exit_val) #once per folder i.e if 3 tests are in a folder and crash, there will be just one exit val
except Exception as e:
print(e)
continue
try:
cur_dir = pathlib.Path(os.path.normpath(dirname)).parent.absolute()
filename_regular_expresion = re.compile('(test_.*)|(TEST_.*)')
for filename in os.listdir(cur_dir):
if filename_regular_expresion.match(filename):
#gets here only if there is a test file which matches the regular expression in the app folder,
#cur_dir = os.path(dirname).parent()
os.chdir(cur_dir)
print(f"Executing coverage test in {cur_dir}")
exit_val = os.system(f"{PY} -m coverage run --append --omit=*/site-packages* -m unittest discover")
microservice_coverage_paths_set.add(os.path.normpath(cur_dir))
except Exception as e:
print(e)
continue
#CHANGE FOLDER TO REPORTS, in order to combine the coverage
try:
if not os.path.exists(REPORTS):
os.makedirs(REPORTS)
except:
pass
try:
os.chdir(REPORTS)
target = REPORTS
target = os.path.normpath( str(target) + f'/.coverage' )
os.remove(target) #Try to Remove old coverage file, if exists
except Exception as e:
pass
print("Combinging coverages")
counter = 0
for path in microservice_coverage_paths_set:
try:
path += '/.coverage'
original = os.path.normpath( path )
target = REPORTS
target = os.path.normpath( str(target) + f'/.coverage.{counter}' )
counter += 1
shutil.copyfile(original,target) #copy new generated coverage files
os.remove(original)
except Exception as e:
print(e)
continue
try:
coverage_xml_path = os.path.normpath( str(REPORTS) + '/coverage.xml')
os.remove(coverage_xml_path)
#coverage_html_path = os.path.normpath( str(REPORTS) + '/htmlcov' )
#os.rmdir(coverage_html_path)
except Exception as e:
print(e)
print("Generating Combined report")
os.system(f"{PY} -m coverage combine")
os.system(f"{PY} -m coverage xml")
os.system(f"{PY} -m coverage html") #if you want to generate the html as well
firstError = -1
i = 0
......@@ -49,3 +128,4 @@ if(firstError<0): #no errors found
sys.exit(0)
else:
sys.exit(1) #return code>0
documentation/external_access.md
View file @ 271733ea
......@@ -31,7 +31,7 @@ This token is used for authentication as _regular user_ on all microservices cur
adds a blockchain transaction entry for ApplicationType with all the keys and values. These will be converted and stored in our own format for creating multilayers and communities.
# Business Logic Microservice
https://articonf1.itec.aau.at:30420/api/ui
https://articonf1.itec.aau.at:30420/api/ui/
This microservice contains use-case specific informations, like schemas and contexts.
......@@ -41,16 +41,54 @@ This microservice contains use-case specific informations, like schemas and cont
## Context information
```GET https://articonf1.itec.aau.at:30420/api/use-cases/{use-case}/layers``` returns all layers from the schema used for clustering interally.
# Trace Retrieval Microservice
https://articonf1.itec.aau.at:30001/api/ui/
This microservice contains the nodes from the transactions preprocessed as defined in *Schema Information*.
```GET https://articonf1.itec.aau.at:30001/api/use_cases/{use_case}/transactions``` returns all flattened transactions, before splitting them into layers.
# Semantic Linking Microservice
https://articonf1.itec.aau.at:30101/api/ui/
This microservice contains the nodes from the transactions preprocessed as defined in *Schema Information*. Additionally it splits the raw input into multipe layers.
This microservice splits the preprocessed transactions into multipe layers, calling the splitted transaction per layer nodes.
```GET https://articonf1.itec.aau.at:30101/api/use-cases/{use-case}/nodes``` returns all preprocessed transactions, called nodes, before splitting them into layers.
```GET https://articonf1.itec.aau.at:30101/api/use-cases/{use_case}/tables/{table_name}/layers/{layer_name}/nodes ``` returns all splitted transactions, called nodes, for the layer layer_name.
# Role Stage Discovery Microservice
https://articonf1.itec.aau.at:30103/api/ui
https://articonf1.itec.aau.at:30103/api/ui/
This microservice contains the communities based on clusters and similarities between communities. It additionally contains time slices with subsets of clusters, which's transaction happened in the corresponding time window.
Schemas and Input data are supplied by the [Business Logic microservice](https://articonf1.itec.aau.at:30420/api/ui), [Semantic Linking microservice](https://articonf1.itec.aau.at:30101/api/ui/) and [Trace Retrieval microservice](https://articonf1.itec.aau.at:30101/api/ui/).
## Layers
Contains information about the schema copied from the Business Logic microservice.
Returns the Schemas and/or Input data used for calculating the clustering which is further used for calculating the similarity.
```GET https://articonf1.itec.aau.at:30103/api/use-cases/{use_case}/layers``` returns layer infos for the given use-case.
```GET https://articonf1.itec.aau.at:30103/api/use-cases/{use_case}/tables/{table}/layers/{layer_name}``` returns the layer information for only the one layer.
```GET https://articonf1.itec.aau.at:30103/api/use-cases/{use_case}/tables/{table}/layers/{layer_name}/nodes``` contains all the nodes contained in the layer fetched from the Semantic Linking microservice.
## Clusters
Contains the clustering results. Clustering is performed on all nodes inside one layer. Furthermore the clusters are partitioned based on timestamps.
```GET https://articonf1.itec.aau.at:30103/api/use-cases/{use_case}/tables/{table}/layers/{layer_name}/clusters``` returns the identified clusters.
```GET https://articonf1.itec.aau.at:30103/api/use-cases/{use_case}/tables/{table}/layers/{layer_name}/timeslices``` returns the identified clusters partitioned based on their nodes' timestamps.
## RunId
When a similarity computation is executed, it has an associated RunId which is used to uniquely identify that execution.
```GET https://articonf1.itec.aau.at:30103/api/runIds``` returns all RunIds in the db.
## Similarity
Returns the computed similarity. Two clusters belonging to the SAME layer will be given a similarity value by comparing them to another cluster belonging to a DIFFERENT layer. This is done for every cluster in the input data. This querry returns all the calculated similarity values, given the criteria (i.e belonging to a use-case,table etc).
```GET https://articonf1.itec.aau.at:30103/api/use_cases/{use_case}/tables/{table}/clusterSimilarity``` returns all similarity values for the given use-case and table.
This microservice contains the communities based on clusters and similarities between communities. It additionally contains time slices with subsets of clusters, which's transaction happened in the corresponding time.
## Connected Cluster
Intermediary data-structure used only by the function which computes the similarity. Clusters are connected only to other clusters belonging to a DIFFERENT layer.
The endpoints are currently refactored, so please check the Swagger UI autogenerated documentation on its website.
\ No newline at end of file
```GET https://articonf1.itec.aau.at:30103/api/use_cases/{use_case}/tables{table}/connectedClusters``` returns all connected clusters for the given use-case and table.
src/data-hub/role-stage-discovery-microservice/app/configs/routes.yml
View file @ 271733ea
......@@ -17,7 +17,7 @@ paths:
description: "Successful echo of request data"
#region Layers
/{use_case}/layers:
/use-cases/{use_case}/layers:
get:
operationId: "routes.layers.get_by_use_case"
security:
......@@ -39,7 +39,7 @@ paths:
'404':
description: "No content for use-case"
/{use_case}/{use_case_table}/layers:
/use-cases/{use_case}/tables/{table}/layers:
get:
operationId: "routes.layers.get_by_table"
security:
......@@ -53,9 +53,9 @@ paths:
description: "Name of the use-case"
required: true
type: "string"
- name: "use_case_table"
- name: "table"
in: "path"
description: "Name of the use-case table"
description: "Name of the table"
required: true
type: "string"
responses:
......@@ -66,7 +66,7 @@ paths:
'404':
description: "No content for use-case"
/{use_case}/{use_case_table}/layers/{name}:
/use-cases/{use_case}/tables/{table}/layers/{layer_name}:
get:
operationId: "routes.layers.get_by_name"
security:
......@@ -80,12 +80,12 @@ paths:
description: "Name of the use-case"
required: true
type: "string"
- name: "use_case_table"
- name: "table"
in: "path"
description: "Name of the use-case table"
description: "Name of the table"
required: true
type: "string"
- name: "name"
- name: "layer_name"
in: "path"
description: "Name of the requested layer"
required: true
......@@ -98,7 +98,7 @@ paths:
'404':
description: "Layer not found"
/{use_case}/{use_case_table}/layers/{name}/nodes:
/use-cases/{use_case}/tables/{table}/layers/{layer_name}/nodes:
get:
operationId: "routes.layers.get_nodes"
security:
......@@ -112,12 +112,12 @@ paths:
description: "Name of the use-case"
required: true
type: "string"
- name: "use_case_table"
- name: "table"
in: "path"
description: "Name of the use-case table"
description: "Name of the table"
required: true
type: "string"
- name: "name"
- name: "layer_name"
in: "path"
description: "Name of the layer"
required: true
......@@ -130,13 +130,13 @@ paths:
'404':
description: "Layer not found"
/{use_case}/{use_case_table}/layers/{name}/clusters:
/use-cases/{use_case}/tables/{table}/layers/{layer_name}/clusters:
get:
operationId: "routes.clustersets.get_by_name"
security:
- JwtRegular: []
tags:
- "Layers"
- "Clusters"
summary: "Get all clusters for the layer"
parameters:
- name: "use_case"
......@@ -144,12 +144,12 @@ paths:
description: "Name of the use-case"
required: true
type: "string"
- name: "use_case_table"
- name: "table"
in: "path"
description: "Name of the use-case table"
description: "Name of the table"
required: true
type: "string"
- name: "name"
- name: "layer_name"
in: "path"
description: "Name of the layer"
required: true
......@@ -162,13 +162,13 @@ paths:
'404':
description: "Layer not found"
/{use_case}/{use_case_table}/layers/{name}/timeslices:
/use-cases/{use_case}/tables{table}/layers/{layer_name}/timeslices:
get:
operationId: "routes.timeslices.get_by_name"
security:
- JwtRegular: []
tags:
- "Layers"
- "Clusters"
summary: "Get all timeslices for the layer"
parameters:
- name: "use_case"
......@@ -176,12 +176,12 @@ paths:
description: "Name of the use-case"
required: true
type: "string"
- name: "use_case_table"
- name: "table"
in: "path"
description: "Name of the use-case table"
description: "Name of the table"
required: true
type: "string"
- name: "name"
- name: "layer_name"
in: "path"
description: "Name of the layer"
required: true
......@@ -196,38 +196,183 @@ paths:
#endregion
#region Similarity
/connectedClusters:
#region Connected
# /connectedClusters:
# get:
# operationId: "routes.connClusters.get_conn_clusters"
# security:
# - JwtRegular: []
# tags:
# - "Connected"
# summary: "Get connected Clusters data"
# description: "Returns a dictionary of cluster. The clusters contain the associated connected clusters and connected nodes data."
# responses:
# '200':
# description: "Successful operation"
# schema:
# $ref: "#/definitions/ConnectedDict"
/use_cases/{use_case}/connectedClusters:
get:
operationId: "routes.connClusters.get_conn_clusters"
operationId: "routes.connClusters.get_conn_clusters_use_case"
security:
- JwtRegular: []
tags:
- "Connected"
summary: "Get connected Clusters data"
- "Connected Cluster"
summary: "Get connected Clusters data for the given Use-Case"
parameters:
- name: "use_case"
in: "path"
description: "Name of the use-case"
required: true
type: "string"
description: "Returns a dictionary of cluster. The clusters contain the associated connected clusters and connected nodes data."
responses:
'200':
description: "Successful operation"
schema:
$ref: "#/definitions/ConnectedDict"
/use_cases/{use_case}/tables/{table}/connectedClusters:
get:
operationId: "routes.connClusters.get_conn_clusters_table"
security:
- JwtRegular: []
tags:
- "Connected Cluster"
summary: "Get connected Clusters data for the given Use-Case and Table"
parameters:
- name: "use_case"
in: "path"
description: "Name of the use-case"
required: true
type: "string"
- name: "table"
in: "path"
description: "Name of the table"
required: true
type: "string"
description: "Returns a dictionary of cluster. The clusters contain the associated connected clusters and connected nodes data."
responses:
'200':
description: "Successful operation"
schema:
$ref: "#/definitions/ConnectedDict"
/use_cases/{use_case}/tables/{table}/layers/{layer_name}/connectedClusters:
get:
operationId: "routes.connClusters.get_conn_clusters_name"
security:
- JwtRegular: []
tags:
- "Connected Cluster"
summary: "Get connected Clusters data for the given Use-Case, Table and Layer"
parameters:
- name: "use_case"
in: "path"
description: "Name of the use-case"
required: true
type: "string"
- name: "table"
in: "path"
description: "Name of the table"
required: true
type: "string"
- name: "layer_name"
in: "path"
description: "Name of the layer"
required: true
type: "string"
description: "Returns a dictionary of cluster. The clusters contain the associated connected clusters and connected nodes data."
responses:
'200':
description: "Successful operation"
schema:
$ref: "#/definitions/ConnectedDict"
#endregion
#region Similarity
/use_cases/{use_case}/batches/{batchNr}/clusterSimilarity:
get:
operationId: "routes.similarity.get_similarity_use_case"
security:
- JwtRegular: []
tags:
- "Similarity"
summary: "Get data of the similarity between clusters given the Use-case."
parameters:
- name: "use_case"
in: "path"
description: "Name of the Use-Case"
required: true
type: "string"
- name: "batchNr"
in: "path"
description: "Batch number (starting from 0)"
required: true
type: "integer"
/clusterSimilarity:
description: "Data is returned in batches of size 1000. Returns a dictionary where the key is a tuple of cluster_labels (i.e. [0,319]) and the value is the computed similarity between 2 clusters in the tuple, in regard to each layer in the input. \n Note: the tuple clusters have the same layer and the computed similarity is in regard to clusters from OTHER layers."
responses:
'200':
description: "Successful operation"
schema:
$ref: "#/definitions/ClusterSimilarityArray"
/use_cases/{use_case}/tables/{table}/batches/{batchNr}/clusterSimilarity:
get:
operationId: "routes.similarity.get_similarity"
operationId: "routes.similarity.get_similarity_table"
security:
- JwtRegular: []
tags:
- "Similarity"
summary: "Get data of the similarity between clusters."
summary: "Get data of the similarity between clusters given the Use-case and Table."
parameters:
- name: "use_case"
in: "path"
description: "Name of the Use-Case"
required: true
type: "string"
- name: "table"
in: "path"
description: "Name of the Table"
required: true
type: "string"
- name: "batchNr"
in: "path"
description: "Batch number (starting from 0)"
required: true
type: "integer"
description: "Data is returned in batches of size 1000. Returns a dictionary where the key is a tuple of cluster_labels (i.e. [0,319]) and the value is the computed similarity between 2 clusters in the tuple, in regard to each layer in the input. \n Note: the tuple clusters have the same layer and the computed similarity is in regard to clusters from OTHER layers."
responses:
'200':
description: "Successful operation"
schema:
$ref: "#/definitions/ClusterSimilarityArray"
/use_cases/{use_case}/tables/{table}/layers/{layer_name}/batches/{batchNr}/clusterSimilarity:
get:
operationId: "routes.similarity.get_similarity_layer"
security:
- JwtRegular: []
tags:
- "Similarity"
summary: "Get data of the similarity between clusters given the Use-case, table and layer name"
parameters:
- name: "use_case"
in: "path"
description: "Name of the Use-Case"
required: true
type: "string"
- name: "table"
in: "path"
description: "Name of the Table"
required: true
type: "string"
- name: "layer_name"
in: "query"
in: "path"
description: "Name of the layer"
required: true
type: "string"
- name: "batchNr"
in: "query"
in: "path"
description: "Batch number (starting from 0)"
required: true
type: "integer"
......@@ -239,15 +384,15 @@ paths:
schema:
$ref: "#/definitions/ClusterSimilarityArray"
/clusterRunArray:
/runIds:
get:
operationId: "routes.connRun.get_connected_run"
security:
- JwtRegular: []
tags:
- "RunId"
- "Similarity"
summary: "Get RunId"
description: "Returns the RunId and the associated datetime when a connection of clusters/simillarity of clusters was computed."
description: "Returns the RunId and the associated datetime when a connection of clusters/similarity of clusters was computed."
responses:
'200':
description: "Successful operation"
......@@ -262,7 +407,7 @@ definitions:
properties:
use_case:
type: string
use_case_table:
table:
type: string
layer_name:
type: string
......@@ -295,7 +440,7 @@ definitions:
type: string
use_case:
type: string
use_case_table:
table:
type: string
LayerCollection:
......@@ -308,7 +453,7 @@ definitions:
example:
"UniqueID": "4437d98b4516e899fb7d93cef0bea6111574473703f0aab9d8c2f02aaa673f5c"
"use_case": "debug"
"use_case_table": "debug-table1"
"table": "debug-table1"
"layer_name": "some_layer"
"some_app_key": "some_app_value"
# "Finished_time": 1576631193265951
......@@ -331,7 +476,7 @@ definitions:
example: "(2020, 52)"
use_case:
type: string
use_case_table:
table:
type: string
layer_name:
type: string
......@@ -366,6 +511,12 @@ definitions:
cluster_label:
type: string
example: "6"
cluster_use_case:
type: string
example: "My_Use_Case"
cluster_table:
type: string
example: "My_Table"
cluster_layer:
type: string
example: "Price_Layer"
......
src/data-hub/role-stage-discovery-microservice/app/db/repository.py
View file @ 271733ea
......@@ -130,7 +130,49 @@ class Repository(MongoRepositoryBase):
if (run_id == None):
entries = super().get_entries(self._connected_clusters_collection, projection={'_id': 0})
else:
entries = super().get_entries(self._similarity_collection, selection={'cluster_runId' : run_id}, projection={'_id': 0})
entries = super().get_entries(self._connected_clusters_collection, selection={'cluster_runId' : run_id}, projection={'_id': 0})
output = []
for ent in entries:
output.append(ent)
return output
# print(ent)
#return [Cluster(cluster_dict=e, from_db=True) for e in entries]
def get_connected_clusters_for_use_case(self,use_case, run_id: str=None):#, layer_name: str):
''' Get Connected Clusters Data given the Use-Case from DB '''
if (run_id == None):
entries = super().get_entries(self._connected_clusters_collection, selection={'cluster_use_case': use_case}, projection={'_id': 0})
else:
entries = super().get_entries(self._connected_clusters_collection, selection={'cluster_runId' : run_id, 'cluster_use_case': use_case}, projection={'_id': 0})
output = []
for ent in entries:
output.append(ent)
return output
# print(ent)
#return [Cluster(cluster_dict=e, from_db=True) for e in entries]
def get_connected_clusters_for_table(self,use_case,table, run_id: str=None):#, layer_name: str):
''' Get Connected Clusters Data given the Use-Case and Table from DB '''
if (run_id == None):
entries = super().get_entries(self._connected_clusters_collection, selection={'cluster_use_case': use_case,'cluster_table': table}, projection={'_id': 0})
else:
entries = super().get_entries(self._connected_clusters_collection, selection={'cluster_runId' : run_id,'cluster_use_case': use_case,'cluster_table': table}, projection={'_id': 0})
output = []
for ent in entries:
output.append(ent)
return output
# print(ent)
#return [Cluster(cluster_dict=e, from_db=True) for e in entries]
def get_connected_clusters_by_name(self,use_case, table, layer_name, run_id: str=None):#, layer_name: str):
''' Get Connected Clusters Data from DB '''
if (run_id == None):
entries = super().get_entries(self._connected_clusters_collection, selection={'cluster_use_case': use_case,'cluster_table': table, 'cluster_layer' : layer_name}, projection={'_id': 0})
else:
entries = super().get_entries(self._connected_clusters_collection, selection={'cluster_runId' : run_id,'cluster_use_case': use_case,'cluster_table': table, 'cluster_layer' : layer_name}, projection={'_id': 0})
output = []
for ent in entries:
......@@ -175,8 +217,38 @@ class Repository(MongoRepositoryBase):
output.append(e)
return output
"""
def get_similarity_use_case(self,skipNr,batchSize,use_case, run_id: str=None):
''' Get Similarity Data from DB '''
if (run_id == None):
entries = super().get_entries(self._similarity_collection, selection={'use_case' : use_case}, projection={'_id': 0})
else:
entries = super().get_entries(self._similarity_collection, selection={'use_case' : use_case, 'runId' : run_id}, projection={'_id': 0})
#
return list(entries.sort([('_id', -1)]).skip(skipNr).limit(batchSize))
def get_similarity_table(self,skipNr,batchSize,use_case,table, run_id: str=None):
''' Get Similarity Data from DB '''
if (run_id == None):
entries = super().get_entries(self._similarity_collection, selection={'use_case' : use_case, 'table': table}, projection={'_id': 0})
else:
entries = super().get_entries(self._similarity_collection, selection={'use_case' : use_case, 'table': table, 'runId' : run_id}, projection={'_id': 0})
#
return list(entries.sort([('_id', -1)]).skip(skipNr).limit(batchSize))
def get_similarity_layer(self,skipNr,batchSize,use_case,table,layer, run_id: str=None):
''' Get Similarity Data from DB '''
if (run_id == None):
entries = super().get_entries(self._similarity_collection, selection={'use_case' : use_case, 'table': table, 'cluster_layer' : layer}, projection={'_id': 0})
else:
entries = super().get_entries(self._similarity_collection, selection={'use_case' : use_case, 'table': table, 'cluster_layer' : layer, 'runId' : run_id}, projection={'_id': 0})
#
return list(entries.sort([('_id', -1)]).skip(skipNr).limit(batchSize))
#endregion
#region connected_run
......
src/data-hub/role-stage-discovery-microservice/app/main.py
View file @ 271733ea
......@@ -18,9 +18,11 @@ from pathlib import Path
from env_info import is_running_locally, get_resources_path
from flask import request
from flask import redirect
from flask_cors import CORS
# load swagger config
app = connexion.App(__name__, specification_dir='configs/')
CORS(app.app)
@app.app.before_request
def before_request():
......
src/data-hub/role-stage-discovery-microservice/app/processing/similarityFiles/dataInput.py
View file @ 271733ea
......@@ -67,7 +67,7 @@ def loadJson(url:str) :
return jsonData
def getClusterDataFromMongo(layerNameList,limitNrCluster,limitNrNodes):
def getClusterDataFromMongo(layerNameList,limitNrCluster,limitNrNodes,use_case,table):
''' Calculates the nr of connections/weights between the clusters contained in the "inputLayerDict". Connections are made between clusters from DIFFERENT layers.
:param List[string] layerNameList: Name of the layers to pull from the DB
......@@ -93,7 +93,7 @@ def getClusterDataFromMongo(layerNameList,limitNrCluster,limitNrNodes):
#imports and translates the data from JSON into usefull format
#returns layerdiction -> Layer -> clusterDict -> Cluster -> nodesDict -> Nodes
for name in layerNameList:
newData = get_mongoDB_cluster_by_layerName(name)#repo.get_clusters_for_layer(name)
newData = get_mongoDB_cluster_by_layerName(use_case,table,name)#repo.get_clusters_for_layer(name)
if newData is not None and len(newData) != 0:
layerDict = populateWithNewNodesSingleLayer(newData[0:limitNrCluster],layerDict,limitNrNodes)
......@@ -290,7 +290,7 @@ def makeChangeNodesDict(inputList,cluster_label,cluster_layer):
outputDict[key]= newNode
return outputDict
def get_mongoDB_cluster_by_layerName(name):
res = repo.get_clusters_for_layer(name)
def get_mongoDB_cluster_by_layerName(use_case, table , layer_name):
res = repo.get_clusters_for_layer(use_case, table, layer_name)
return [c.to_serializable_dict() for c in res]
src/data-hub/role-stage-discovery-microservice/app/processing/similarityFiles/dataOutput.py
View file @ 271733ea
......@@ -6,7 +6,7 @@ from processing.similarityFiles.miscFunctions import *
from db.repository import Repository
repo = Repository()
def outputFileLayerFunction(layerDict,limitNrNodes,limitNrCluster,runId):
def outputFileLayerFunction(layerDict,limitNrNodes,limitNrCluster,runId,table,use_case):
''' Writes the layerDict data to a JSON file.
:param Dict{string: Layer} layerDict: Object which contains Data about the Layers, Clusters and Nodes
......@@ -17,7 +17,7 @@ def outputFileLayerFunction(layerDict,limitNrNodes,limitNrCluster,runId):
'''
layerJSON = convertLayerDictToJSON(layerDict,runId)
layerJSON = convertLayerDictToJSON(layerDict,runId,table,use_case)
outputJSON = json.dumps(layerJSON, default=lambda o: o.__dict__, indent=4)
try:
......@@ -28,7 +28,7 @@ def outputFileLayerFunction(layerDict,limitNrNodes,limitNrCluster,runId):
def outputFileSimilFunction(similarityDict,limitNrNodes,limitNrCluster,runId):
def outputFileSimilFunction(similarityDict,limitNrNodes,limitNrCluster,runId,table,use_case):
''' Writes the similarityDict data to a JSON file.
......@@ -40,7 +40,7 @@ def outputFileSimilFunction(similarityDict,limitNrNodes,limitNrCluster,runId):
'''
similJSON = convertSimilarityDictToJSON(similarityDict,runId)
similJSON = convertSimilarityDictToJSON(similarityDict,runId,table,use_case)
outputJSON = json.dumps(similJSON, default=lambda o: o.__dict__, indent=4)
try:
......@@ -77,7 +77,7 @@ def outputFileTimeFunction(timelist,limitNrNodes,limitNrCluster,runId):
print("Error occured when writing the resultTimeExec file")
def outputMongoConnClustDict(inputDict,runId):
def outputMongoConnClustDict(inputDict,runId,table,use_case):
''' Stores connected_clusters in the database.
......@@ -89,9 +89,9 @@ def outputMongoConnClustDict(inputDict,runId):
#inputDict["Timestamp"] = str(datetime.datetime.now())
add_conn_clusters(inputDict,runId)
add_conn_clusters(inputDict,runId,table,use_case)
def outputMongoSimilarity(inputDict,runId):
def outputMongoSimilarity(inputDict,runId,table,use_case):
''' Stores cluster_similarity in the database.
:param Dict() inputDict: Contains the data to insert
......@@ -99,7 +99,7 @@ def outputMongoSimilarity(inputDict,runId):
:param string runId: Id of the Run
'''
add_similarity(inputDict,runId)
add_similarity(inputDict,runId,table,use_case)
def add_connected_run():
......@@ -116,7 +116,7 @@ def add_connected_run():
inserted_result = repo.add_connected_run(runDict)
return str(inserted_result.inserted_id)
def add_conn_clusters(inputDict,runId):
def add_conn_clusters(inputDict,runId,table,use_case):
''' Stores connected_clusters in the database.
:param Dict() inputDict: Contains the data to insert
......@@ -125,11 +125,11 @@ def add_conn_clusters(inputDict,runId):
'''
outputJSON = convertLayerDictToJSON(inputDict,runId)
outputJSON = convertLayerDictToJSON(inputDict,runId,table,use_case)
for element in outputJSON:
repo.add_connected_cluster(element)
def add_similarity(inputDict,runId):
def add_similarity(inputDict,runId,table,use_case):
''' Stores cluster_similarity in the database.
:param Dict() inputDict: Contains the data to insert
......@@ -138,6 +138,6 @@ def add_similarity(inputDict,runId):
'''
outputJSON = convertSimilarityDictToJSON(inputDict,runId)
outputJSON = convertSimilarityDictToJSON(inputDict,runId,table,use_case)
for element in outputJSON:
repo.add_single_similarity(element)
\ No newline at end of file
src/data-hub/role-stage-discovery-microservice/app/processing/similarityFiles/miscFunctions.py
View file @ 271733ea
......@@ -42,7 +42,7 @@ def totalNumberOfClusters(inputLayerDict):
return clustCount
def convertLayerDictToJSON(layerDict, runId):
def convertLayerDictToJSON(layerDict, runId,table,use_case):
''' Converts a Layer object to JSON format.
:param Dict{string: Layer} layerDict: Object which contains Data about the Layers, Clusters and Nodes
......@@ -57,6 +57,8 @@ def convertLayerDictToJSON(layerDict, runId):
outputJSON.append({
"cluster_label" : curCluster.cluster_label,
"cluster_layer" : curCluster.cluster_layer,
"cluster_table" : table,
"cluster_use_case": use_case,
"cluster_runId" : runId,
"cluster_connClustDict" : changeTupleDictToDictList(curCluster.cluster_connClustDict),
"cluster_connNodesDict" : getFrozensetFromConnNodesDict(curCluster.cluster_connNodesDict), #Don
......@@ -109,7 +111,7 @@ def getFrozensetFromConnNodesDict(inputDict):
return output
def convertSimilarityDictToJSON(inputDict,runId):
def convertSimilarityDictToJSON(inputDict,runId,table,use_case):
''' Converts a Similarity Dictionary to JSON format. For outputting to DB
:param Dict{} similarityDict: Object which contains Data about the Computed similarities between Clusters
......@@ -125,6 +127,8 @@ def convertSimilarityDictToJSON(inputDict,runId):
auxDict["cluster_layer"] = tupleKey[2]
auxDict["similarityValues"] = inputDict[tupleKey]
auxDict["runId"] = runId
auxDict["table"] = table
auxDict["use_case"] = use_case
similList.append(auxDict)
similToJSON = similList
#outputJSON = json.dumps(similToJSON, default=lambda o: o.__dict__, indent=4)
......
src/data-hub/role-stage-discovery-microservice/app/processing/similarityFiles/similarityMain.py
View file @ 271733ea
......@@ -39,7 +39,7 @@ from processing.similarityFiles.dataOutput import *
outputToFileFLAG = True
def main(layerNameList:List[str] = ["Price_Layer","FinishedTime_Layer","Destination_Layer"]):
def main(layerNameList:List[str] , table:str , use_case: str):
'''
Executes the similarity calculation by calculating weights between clusters in different layers.
Then calculating the Euclidean distance between nodes in the same layer based on one other layer each.
......@@ -48,7 +48,8 @@ def main(layerNameList:List[str] = ["Price_Layer","FinishedTime_Layer","Destinat
:param layerNameList: The list of layer names as strings
'''
print("Entered Similarity Main")
if len(layerNameList)==0:
return
timelist = []
timelist.append(currentTime())#starting time
......@@ -67,7 +68,7 @@ def main(layerNameList:List[str] = ["Price_Layer","FinishedTime_Layer","Destinat
limitNrNodes = -1 #per Layer
layerDict = getClusterDataFromMongo(layerNameList,limitNrCluster,limitNrNodes)
layerDict = getClusterDataFromMongo(layerNameList,limitNrCluster,limitNrNodes,use_case,table)
if layerDict is None or len(layerDict) == 0:
LOGGER.error(f"No data for any of the following layers existed: {str(layerNameList)}. Similarity calculation was not performed.")
return
......@@ -98,13 +99,13 @@ def main(layerNameList:List[str] = ["Price_Layer","FinishedTime_Layer","Destinat
if (outputToFileFLAG == True):
print("Outputing data")
outputFileLayerFunction(layerDict,totalNodes,totalClusters,runId)
outputFileSimilFunction(similarityDict,totalNodes,totalClusters,runId)
outputFileLayerFunction(layerDict,totalNodes,totalClusters,runId,table,use_case)
outputFileSimilFunction(similarityDict,totalNodes,totalClusters,runId,table,use_case)
outputFileTimeFunction(timelist,totalNodes,totalClusters,runId)
#Output to DB
outputMongoConnClustDict(layerDict,runId)
outputMongoSimilarity(similarityDict,runId)
outputMongoConnClustDict(layerDict,runId,table,use_case)
outputMongoSimilarity(similarityDict,runId,table,use_case)
#Currently not used in the calculation of connections/similarity, developed for possible future uses
......@@ -122,6 +123,6 @@ def main(layerNameList:List[str] = ["Price_Layer","FinishedTime_Layer","Destinat
return
##########START##########
if __name__ is '__main__':
main()
#if __name__ is '__main__':
#main()
#########FINISH##########
src/data-hub/role-stage-discovery-microservice/app/requirements.txt
View file @ 271733ea
......@@ -8,11 +8,13 @@ Click==7.0
clickclick==1.2.2
colorama==0.4.3
connexion==2.6.0
coverage==5.3.1
cryptography==3.1
cycler==0.10.0
decorator==4.4.1
Deprecated==1.2.7
Flask==1.1.1
Flask-Cors==3.0.10
idna==2.8
importlib-metadata==1.5.0
inflection==0.3.1
......
src/data-hub/role-stage-discovery-microservice/app/routes/clustersets.py
View file @ 271733ea
......@@ -4,8 +4,8 @@ from db.entities import ClusterSet
repo = Repository()
def get_by_name(use_case, use_case_table, name):
res = repo.get_clusters_for_layer(use_case, use_case_table, name)
def get_by_name(use_case, table, layer_name):
res = repo.get_clusters_for_layer(use_case, table, layer_name)
if res is None or len(res) == 0:
return Response(status=404)
else:
......
src/data-hub/role-stage-discovery-microservice/app/routes/connClusters.py
View file @ 271733ea
......@@ -16,3 +16,45 @@ def get_conn_clusters():
else:
return result
def get_conn_clusters_use_case(use_case):
''' Gets connected_clusters from the database.
:returns: Returns similarity objects from the DB
:rtype: Dict
'''
result = repo.get_connected_clusters_for_use_case(use_case)
if result is None or len(result) == 0:
print("MongoDb Get Error: Response 404")
return Response(status=404)
else:
return result
def get_conn_clusters_table(use_case,table):
''' Gets connected_clusters from the database.
:returns: Returns similarity objects from the DB
:rtype: Dict
'''
result = repo.get_connected_clusters_for_table(use_case, table)
if result is None or len(result) == 0:
print("MongoDb Get Error: Response 404")
return Response(status=404)
else:
return result
def get_conn_clusters_name(use_case,table,layer_name):
''' Gets connected_clusters from the database.
:returns: Returns similarity objects from the DB
:rtype: Dict
'''
result = repo.get_connected_clusters_by_name(use_case,table,layer_name)
if result is None or len(result) == 0:
print("MongoDb Get Error: Response 404")
return Response(status=404)
else:
return result
src/data-hub/role-stage-discovery-microservice/app/routes/layers.py
View file @ 271733ea
......@@ -26,15 +26,15 @@ def get_by_use_case(use_case):
else:
return Response(status=404)
def get_by_table(use_case, use_case_table):
res = repo.get_layers_for_table(use_case, use_case_table)
def get_by_table(use_case, table):
res = repo.get_layers_for_table(use_case, table)
if len(res) > 0:
return [l.to_serializable_dict() for l in res]
else:
return Response(status=404)
def get_by_name(use_case, use_case_table, name):
res = repo.get_layer_by_name(use_case, use_case_table, name)
def get_by_name(use_case, table, layer_name):
res = repo.get_layer_by_name(use_case, table, layer_name)
if res is not None:
return res.to_serializable_dict()
else:
......@@ -43,8 +43,8 @@ def get_by_name(use_case, use_case_table, name):
#endregion
#region nodes
def get_nodes(use_case, use_case_table, name):
res = repo.get_layer_nodes(use_case, use_case_table, name)
def get_nodes(use_case, table, layer_name):
res = repo.get_layer_nodes(use_case, table, layer_name)
# print(res)
return res
......
src/data-hub/role-stage-discovery-microservice/app/routes/similarity.py
View file @ 271733ea
......@@ -23,3 +23,60 @@ def get_similarity(layer_name,batchNr):
return Response(status=404)
else:
return result
def get_similarity_use_case(use_case,batchNr):
''' Gets cluster_similarity from the database.
:returns: Returns similarity objects from the DB
:rtype: Dict
'''
batchSize = 1000
if int(batchNr)<0:
print("Batch number needs to be a positive integer")
return Response(status=404)
skipNr = batchSize*int(batchNr)
#get_similarity(self,skipNr,batchSize, cluster_layer: str= None, run_id: str=None)
result = repo.get_similarity_use_case(skipNr, batchSize, use_case)
if result is None or len(result) == 0:
print("MongoDb Get Error: Response 404")
return Response(status=404)
else:
return result
def get_similarity_table(use_case,table,batchNr):
''' Gets cluster_similarity from the database.
:returns: Returns similarity objects from the DB
:rtype: Dict
'''
batchSize = 1000
if int(batchNr)<0:
print("Batch number needs to be a positive integer")
return Response(status=404)
skipNr = batchSize*int(batchNr)
#get_similarity(self,skipNr,batchSize, cluster_layer: str= None, run_id: str=None)
result = repo.get_similarity_table(skipNr, batchSize, use_case,table)
if result is None or len(result) == 0:
print("MongoDb Get Error: Response 404")
return Response(status=404)
else:
return result
def get_similarity_layer(use_case,table,layer_name,batchNr):
''' Gets cluster_similarity from the database.
:returns: Returns similarity objects from the DB
:rtype: Dict
'''
batchSize = 1000
if int(batchNr)<0:
print("Batch number needs to be a positive integer")
return Response(status=404)
skipNr = batchSize*int(batchNr)
#get_similarity(self,skipNr,batchSize, cluster_layer: str= None, run_id: str=None)
result = repo.get_similarity_layer(skipNr, batchSize,use_case,table, layer_name)
if result is None or len(result) == 0:
print("MongoDb Get Error: Response 404")
return Response(status=404)
else:
return result
src/data-hub/role-stage-discovery-microservice/app/routes/timeslices.py
View file @ 271733ea
......@@ -4,8 +4,8 @@ from db.entities import TimeSlice
repo = Repository()
def get_by_name(use_case, use_case_table, name):
res = repo.get_time_slices_by_name(use_case, use_case_table, name)
def get_by_name(use_case, table, layer_name):
res = repo.get_time_slices_by_name(use_case, table, layer_name)
if res is not None and len(res) != 0:
return [e.to_serializable_dict() for e in res]
......
src/data-hub/role-stage-discovery-microservice/app/run_similarity_calc.py
View file @ 271733ea
......@@ -7,17 +7,38 @@ repo = Repository()
def run_similarity_calc_per_use_case():
layers = repo.get_layers()
uc_layers = {}
# uc_layers = {}
# for layer in layers:
# uc = layer.use_case
# if uc not in uc_layers:
# uc_layers[uc] = []
# uc_layers[uc].append(layer.layer_name)
# for key in uc_layers:
# layers2 = uc_layers[key]
# print(f"Running for use case {key} with layers {str(layers2)}.")
# SimilarityCalc.main(layerNameList=layers2)
uc_dict = dict()
# use_case[table[layer_name]]
for layer in layers:
uc = layer.use_case
if uc not in uc_layers:
uc_layers[uc] = []
uc_layers[uc].append(layer.layer_name)
for key in uc_layers:
layers = uc_layers[key]
print(f"Running for use case {key} with layers {str(layers)}.")
SimilarityCalc.main(layerNameList=layers)
use_case = layer.use_case
table = layer.use_case_table
if use_case not in uc_dict:
uc_dict[use_case] = dict()
#aux = uc_dict[use_case]
if table not in uc_dict[use_case]:
uc_dict[use_case][table] = []
uc_dict[use_case][table].append(layer.layer_name)
for uc in uc_dict:
for table in uc_dict[uc]:
layers2 = uc_dict[uc][table]
print(f"Running for use case {uc}, table {table}, with layers {str(layers2)}.")
SimilarityCalc.main(layers2,table,uc)
if __name__ == '__main__':
......
src/data-hub/role-stage-discovery-microservice/app/test_coverage.py 0 → 100644
View file @ 271733ea
import unittest
import sys
for path in ['../', './']:
sys.path.insert(1, path)
#####################################
### Don't include for test report ###
#####################################
try:
class TestCoverage(unittest.TestCase):
def test_init_main(self):
try:
# python -m unittest discover
from db.entities import Cluster
from datetime import date, datetime
import json
# add modules folder to interpreter path
import sys
import os
modules_path = '../../../modules/'
if os.path.exists(modules_path):
sys.path.insert(1, modules_path)
### init logging ###
import logging
LOG_FORMAT = ('%(levelname) -5s %(asctime)s %(name)s:%(funcName) -35s %(lineno) -5d: %(message)s')
logging.basicConfig(level=logging.INFO, format=LOG_FORMAT)
LOGGER = logging.getLogger(__name__)
#############################
import connexion
from security import swagger_util
from pathlib import Path
from env_info import is_running_locally, get_resources_path
from flask import request
from flask import redirect
except Exception as e:
print ("Exception found:")
print (e)
try:
import main #error when importing main, ModuleNotFoundError: No module named 'security'
#exec(open('main.py').read())
except Exception as e:
print ("Exception found:")
print (e)
def test_init_run_clustering(self):
try:
import sys
import os
modules_path = '../../../modules/'
if os.path.exists(modules_path):
sys.path.insert(1, modules_path)
import json
from db.entities import Layer, Cluster
from typing import List, Dict, Tuple, Any
from db.repository import Repository
from processing.clustering import Clusterer, ClusterResult
except Exception as e:
print ("Exception found:")
print (e)
try:
import run_clustering
except Exception as e:
print ("Exception found:")
print (e)
def test_init_run_node(self):
try:
import sys
import os
modules_path = '../../../modules/'
if os.path.exists(modules_path):
sys.path.insert(1, modules_path)
import json
import urllib3
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
except Exception as e:
print ("Exception found:")
print (e)
try:
import processing.fetching.fetching as f
import run_node_fetching
except Exception as e:
print ("Exception found:")
print (e)
def test_init_run_similarity(self):
try:
import processing.similarityFiles.similarityMain as SimilarityCalc
from db.repository import Repository
import run_similarity_calc
except Exception as e:
print ("Exception found:")
print (e)
def test_init_run_time(self):
try:
import sys
import os
modules_path = '../../../modules/'
if os.path.exists(modules_path):
sys.path.insert(1, modules_path)
import json
from datetime import datetime, date
from db.repository import Repository
from db.entities import ClusterSet, Cluster, Layer, TimeSlice
from typing import Tuple, Dict, Any, List
except Exception as e:
print ("Exception found:")
print (e)
try:
import run_time_slicing
except Exception as e:
print ("Exception found:")
print (e)
if __name__ == '__main__':
unittest.main()
except Exception as e:
print ("Exception found:")
print (e)
\ No newline at end of file
src/data-hub/role-stage-discovery-microservice/app/tests/test_cluster.py
View file @ 271733ea
import unittest
import sys
for path in ['../', './']:
......@@ -19,6 +20,5 @@ class TestCluster(unittest.TestCase):
self.assertEqual(1, c.cluster_label)
self.assertEqual([1, 2, 3], c.nodes)
if __name__ == '__main__':
unittest.main()
src/data-hub/role-stage-discovery-microservice/app/tests/test_cluster_result.py
View file @ 271733ea
......@@ -11,8 +11,10 @@ class TestClusterResult(unittest.TestCase):
converter:ClusterResultConverter = None
def setUp(self):
self.converter = ClusterResultConverter()
def test_result_undefined_feature(self):
cluster_groups = self._get_some_cluster_groups_1d()
cluster_res = self.converter.convert_to_cluster_results(
......@@ -30,10 +32,12 @@ class TestClusterResult(unittest.TestCase):
features=['v']
)
self.assert_correct_cluster_result_len(cluster_groups, cluster_res)
self.assert_correct_cluster_result_labels(['-1.0 -- 1.0','10.0 -- 11.0','2.0 -- 2.0'], cluster_res)
def test_result_2d_features(self):
cluster_groups = self._get_some_cluster_groups_2d()
cluster_res = self.converter.convert_to_cluster_results(
cluster_groups=cluster_groups,
......@@ -44,6 +48,7 @@ class TestClusterResult(unittest.TestCase):
self.assert_correct_cluster_result_labels([str((0.0,0.0)), str((10.5,10.5)), str((2.0,2.0)), str((3.0,6.0))], cluster_res)
#region Custom Assertions
def assert_correct_cluster_result_len(self, expected: 'original dict of lists', actual: Dict[Any, ClusterResult]):
......@@ -52,6 +57,7 @@ class TestClusterResult(unittest.TestCase):
self.assertEqual(len(expected[i]), len(actual[i].nodes))
self.assertEqual(expected[i], actual[i].nodes)
def assert_correct_cluster_result_labels(self, expected: List[str], actual: Dict[Any, ClusterResult]):
self.assertEqual(len(expected), len(actual))
for i in range(len(expected)):
......
src/data-hub/role-stage-discovery-microservice/app/tests/test_clusterer.py
View file @ 271733ea
import unittest
import sys
for path in ['../', './']:
sys.path.insert(1, path)
# python -m unittest discover
from processing.clustering import Clusterer, ClusterResult
import numpy as np
......@@ -192,3 +195,4 @@ class TestClusterer(unittest.TestCase):
if __name__ == '__main__':
unittest.main()
src/data-hub/role-stage-discovery-microservice/app/tests/test_similarity.py
View file @ 271733ea
......@@ -23,7 +23,7 @@ import json
class TestSimilarity(unittest.TestCase):
'''Tests the similarity calculation which works without object orientation.'''
def test_integration_similarityCalculation(self):
def test_integration_calculateSimilarity_ClustersDict_CorrectValue(self):
'''
Only for testing, can be deleted at any time.\n
Served as a testing example to make sure the computations are correct
......
src/data-hub/semantic-linking-microservice/app/dummy_upload/smart_energy/dummy_upload.py
View file @ 271733ea
......@@ -9,12 +9,14 @@ for modules_path in modules_paths:
sys.path.insert(1, modules_path)
from messaging.MessageHandler import MessageHandler
from db.repository import Repository
# file to read the data from
CSV_FILE = r'Energy_Dataset.csv'
handler = MessageHandler()
CSV_FILE = r'dummy_upload\smart_energy\Energy_Dataset.csv'
handler = MessageHandler(Repository())
processed_transactions = []
def upload_transaction(transaction):
'''{"type": "new-trace",
"content": {"use_case": "smart-energy", "table": "smart-energy", "id": "dd2c5146c919b046d77a32a5cf553d5133163562f7b7e1298c878c575d516025",
......@@ -28,7 +30,31 @@ def upload_transaction(transaction):
'id': uid,
'properties': transaction,
}
handler.handle_new_trace(t)
# handler.handle_new_trace(t)
processed_transactions.append(t)
def store_transactions_for_mirsat():
'''
Stores the processed transactions as if they would be returned
by Trace Retrieval microservice after fixing the message queue bug.
'''
flattened_transactions = []
for transaction in processed_transactions:
transaction = transaction['properties']
transaction['use_case'] = transaction['ApplicationType']
del transaction['ApplicationType']
transaction['table'] = transaction['docType']
del transaction['docType']
flattened_transactions.append(transaction)
import json
with open('flattened_smart_energy_data.json', 'w') as file:
file.write(json.dumps(flattened_transactions))
if __name__ == '__main__':
......@@ -56,3 +82,4 @@ if __name__ == '__main__':
upload_transaction(transaction)
store_transactions_for_mirsat()
\ No newline at end of file
src/data-hub/semantic-linking-microservice/app/main.py
View file @ 271733ea
......@@ -20,6 +20,7 @@ from env_info import is_running_locally, get_resources_path
from messaging.ReconnectingMessageManager import ReconnectingMessageManager
from messaging.MessageHandler import MessageHandler
from flask import request
from flask_cors import CORS
from flask import redirect
# init message handler
......@@ -30,6 +31,7 @@ def message_received_callback(channel, method, properties, body):
# load swagger config
app = connexion.App(__name__, specification_dir='configs/')
CORS(app.app)
@app.app.before_request
def before_request():
......
src/data-hub/semantic-linking-microservice/app/messaging/MessageHandler.py
View file @ 271733ea
......@@ -90,6 +90,7 @@ class MessageHandler:
for prop in layer.total_properties:
node[prop] = content["properties"][prop]
node["layer_name"] = layer.layer_name
node["table"] = layer.table
node["use_case"] = layer.use_case
......
src/data-hub/semantic-linking-microservice/app/requirements.txt
View file @ 271733ea
......@@ -7,8 +7,10 @@ click==7.1.2
clickclick==1.2.2
colorama==0.4.3
connexion==2.7.0
coverage==5.3.1
cryptography==3.1
Flask==1.1.2
Flask-Cors==3.0.10
idna==2.9
importlib-metadata==1.6.1
inflection==0.5.0
......
src/data-hub/semantic-linking-microservice/app/test_coverage.py 0 → 100644
View file @ 271733ea
import unittest
import sys
for path in ['../', './']:
sys.path.insert(1, path)
#####################################
### Don't include for test report ###
#####################################
try:
class TestCoverage(unittest.TestCase):
def test_init_main(self):
try:
# add modules folder to interpreter path
import sys
import os
import prance
from pathlib import Path
modules_path = '../../../modules/'
if os.path.exists(modules_path):
sys.path.insert(1, modules_path)
# init logging to file
import logging
LOG_FORMAT = ('%(levelname) -5s %(asctime)s %(name)s:%(funcName) -35s %(lineno) -5d: %(message)s')
logging.basicConfig(level=logging.INFO, format=LOG_FORMAT)
LOGGER = logging.getLogger(__name__)
#################################
import connexion
from security import swagger_util
from env_info import is_running_locally, get_resources_path
from messaging.ReconnectingMessageManager import ReconnectingMessageManager
from messaging.MessageHandler import MessageHandler
from flask import request
from flask import redirect
# init message handler
from db.repository import Repository
except Exception as e:
print ("Exception found:")
print (e)
try:
import main
except Exception as e:
print ("Exception found:")
print (e)
def test_routes(self):
try:
from routes import debug
except Exception as e:
print ("Exception found:")
print (e)
try:
from routes import layers
except Exception as e:
print ("Exception found:")
print (e)
try:
from routes import nodes
except Exception as e:
print ("Exception found:")
print (e)
def test_messaging(self):
try:
import network_constants as netconst
from security.token_manager import TokenManager
from db.entities import Layer
import json
import requests
from typing import Dict, List
from threading import Thread
import logging
except Exception as e:
print ("Exception found:")
print (e)
try:
from messaging import MessageHandler
except Exception as e:
print ("Exception found:")
print (e)
def test_db(self):
try:
import network_constants as netconst
from database.MongoRepositoryBase import MongoRepositoryBase
from db.entities import Layer
import pymongo
import json
from typing import List, Dict
# init logging to file
import logging
except Exception as e:
print ("Exception found:")
print (e)
try:
from db import repository
from db.entities import layer
except Exception as e:
print ("Exception found:")
print (e)
if __name__ == '__main__':
unittest.main()
except Exception as e:
print ("Exception found:")
print (e)
\ No newline at end of file
src/data-hub/semantic-linking-microservice/app/tests/test_pipeline.py
View file @ 271733ea
......@@ -11,6 +11,7 @@ from typing import List
class DummyMongoRepo:
'''Dummy class to be used for testing the MessageHandler'''
last_trace = None
layernodes = []
def insert_trace(self, trace):
self.last_trace = trace
......@@ -34,7 +35,9 @@ class DummyMongoRepo:
]
def add_layer_nodes(self, nodes: List):
pass
self.layernodes.extend(nodes)
return
class Test_Pipeline(unittest.TestCase):
handler = None
......@@ -64,10 +67,11 @@ class Test_Pipeline(unittest.TestCase):
}
}
}
def testTraceProcessing(self):
#original name testTraceProcessing
def test_handle_new_trace_newTraceMsg_correctlyInserted(self):
msg = self._buildTraceMessage()
self.handler.handle_new_trace(msg["content"])
self.assertEqual(len(self.handler._repository.layernodes),1)
if __name__ == '__main__':
......
src/modules/messaging/MessageManager.py
View file @ 271733ea
import pika
from pika import spec
from threading import Thread
import network_constants as netconst
......@@ -47,10 +48,18 @@ class MessageManager:
if self._error_callback != None:
self._error_callback("Connection closed")
def _message_confirmed_callback(self, frame):
if isinstance(frame.method, spec.Basic.Ack):
LOGGER.debug("message acknowledged")
return
else:
LOGGER.warning("Message was rejected by broker!")
def _channel_created_callback(self, channel: pika.channel.Channel):
# Assign both channels
if self._send_channel == None:
self._send_channel = channel
self._send_channel.confirm_delivery(ack_nack_callback=self._message_confirmed_callback)
else:
self._receive_channel = channel
LOGGER.info("RabbitMQ connection established")
......@@ -91,7 +100,10 @@ class MessageManager:
def send_message(self, exchange_name, routing_key, message):
'''Sends a message to the exchange'''
self._send_channel.basic_publish(exchange_name, routing_key, message)
try:
self._send_channel.basic_publish(exchange_name, routing_key, message, mandatory=True)
except pika.exceptions.UnroutableError:
raise pika.exceptions.UnroutableError
def disconnect(self):
'''Stops listening for messages and closes the connection'''
......
src/modules/messaging/ReconnectingMessageManager.py
View file @ 271733ea
......@@ -3,6 +3,8 @@ import time
from messaging.MessageManager import MessageManager
import logging
import json
import pika
LOGGER = logging.getLogger(__name__)
class ReconnectingMessageManager:
......@@ -91,5 +93,13 @@ class ReconnectingMessageManager:
'''Sends a message to the exchange'''
try:
self._message_manager.send_message(exchange_name, routing_key, message)
except pika.exceptions.UnroutableError:
message = json.loads(message)
if message["retries"] < 10:
message["retries"] += 1
message = json.dumps(message)
self.send_message(exchange_name, routing_key, message)
else:
LOGGER.error("Message went through too many send retries and was not delivered")
except:
LOGGER.error("Error while sending message")
src/participation-hub/business-logic-microservice/app/_add_use_case_scripts/car_sharing/tables/add_car.py
View file @ 271733ea
from _add_use_case_scripts.car_sharing.tables.requestPost import postLayersToSwagger, postTableToSwagger
from _add_use_case_scripts.requestPost import postLayersToSwagger, postTableToSwagger
def add_table(use_case: str, table_name:str):
'''
......
src/participation-hub/business-logic-microservice/app/_add_use_case_scripts/car_sharing/tables/add_hash.py
View file @ 271733ea
from _add_use_case_scripts.car_sharing.tables.requestPost import postLayersToSwagger, postTableToSwagger
from _add_use_case_scripts.requestPost import postLayersToSwagger, postTableToSwagger
def add_table(use_case: str, table_name: str):
'''
......
src/participation-hub/business-logic-microservice/app/_add_use_case_scripts/car_sharing/tables/add_offer.py
View file @ 271733ea
from _add_use_case_scripts.car_sharing.tables.requestPost import postLayersToSwagger, postTableToSwagger
from _add_use_case_scripts.requestPost import postLayersToSwagger, postTableToSwagger
def add_table(use_case: str, table_name: str):
'''
......
src/participation-hub/business-logic-microservice/app/_add_use_case_scripts/car_sharing/tables/add_publication.py
View file @ 271733ea
from _add_use_case_scripts.car_sharing.tables.requestPost import postLayersToSwagger, postTableToSwagger
from _add_use_case_scripts.requestPost import postLayersToSwagger, postTableToSwagger
def add_table(use_case: str, table_name: str):
'''
......
src/participation-hub/business-logic-microservice/app/_add_use_case_scripts/car_sharing/tables/add_travel.py
View file @ 271733ea
from _add_use_case_scripts.car_sharing.tables.requestPost import postLayersToSwagger, postTableToSwagger
from _add_use_case_scripts.requestPost import postLayersToSwagger, postTableToSwagger
def add_table(use_case: str, table_name: str):
'''
......
src/participation-hub/business-logic-microservice/app/_add_use_case_scripts/car_sharing/tables/add_user.py
View file @ 271733ea
from _add_use_case_scripts.car_sharing.tables.requestPost import postLayersToSwagger, postTableToSwagger
from _add_use_case_scripts.requestPost import postLayersToSwagger, postTableToSwagger
def add_table(use_case: str, table_name: str):
......
src/participation-hub/business-logic-microservice/app/_add_use_case_scripts/crowd_journalism/add_crowdjournalism_schema.py 0 → 100644
View file @ 271733ea
import sys
import os
from pathlib import Path
from typing import Dict, Any
import requests
modules_paths = ['.', '../../../modules/']
for modules_path in modules_paths:
if os.path.exists(modules_path):
sys.path.insert(1, modules_path)
from _add_use_case_scripts.crowd_journalism.tables import add_video,add_tag,add_purchase,add_event,add_classification
import network_constants as nc
from security.token_manager import TokenManager
def add_use_case(use_case: str):
jwt = TokenManager.getInstance().getToken()
url = f"https://articonf1.itec.aau.at:30420/api/use-cases"
response = requests.post(
url,
verify=False,
proxies = { "http":None, "https":None },
headers = { "Authorization": f"Bearer {jwt}"},
json = {"name": use_case}
)
print(url+": "+str(response.content))
if __name__ == "__main__":
use_case = "crowd-journalism"
# disable ssl warnings :)
requests.packages.urllib3.disable_warnings()
add_use_case(use_case)
add_video.main(use_case)
add_tag.main(use_case)
add_classification.main(use_case)
add_event.main(use_case)
add_purchase.main(use_case)
\ No newline at end of file
src/participation-hub/business-logic-microservice/app/_add_use_case_scripts/crowd_journalism/tables/add_classification.py 0 → 100644
View file @ 271733ea
from _add_use_case_scripts.requestPost import postLayersToSwagger, postTableToSwagger
def add_table(use_case: str, table_name: str):
'''
take the columns and add the mappings at the server
replace all "/"'s in the internal representation with a "_"
'''
columns = [
# "docType",
"objecttype",
"userid",
"videoid",
"informative",
"impact",
"trustiness",
"lastupdate"
]
columns = { c : c for c in columns }
columns["UniqueID"] = "userid+videoid"
table = {
"name": table_name,
"mappings": columns
}
postTableToSwagger(use_case,table)
def add_layers(use_case:str, table_name: str):
layers = [
{ #Useless as all objects aree Classification?
"use_case": use_case,
"table": table_name,
"name": "Object_Type_Layer",
"properties": [
"UniqueID",
"objecttype",
"userid",
"videoid",
"informative",
"impact",
"trustiness",
"lastupdate"
],
"cluster_properties": [
"objecttype"
]
},
{
"use_case": use_case,
"table": table_name,
"name": "Informative_Layer",
"properties": [
"UniqueID",
"objecttype",
"userid",
"videoid",
"informative",
"impact",
"trustiness",
"lastupdate"
],
"cluster_properties": [
"informative"
]
},
{
"use_case": use_case,
"table": table_name,
"name": "Impact_Layer",
"properties": [
"UniqueID",
"objecttype",
"userid",
"videoid",
"informative",
"impact",
"trustiness",
"lastupdate"
],
"cluster_properties": [
"impact"
]
},
{
"use_case": use_case,
"table": table_name,
"name": "Trust_Layer",
"properties": [
"UniqueID",
"objecttype",
"userid",
"videoid",
"informative",
"impact",
"trustiness",
"lastupdate"
],
"cluster_properties": [
"trustiness"
]
}
]
postLayersToSwagger(use_case,layers)
def main(use_case: str):
print("Classification")
table_name = "classification"
add_table(use_case,table_name)
add_layers(use_case,table_name)
\ No newline at end of file
src/participation-hub/business-logic-microservice/app/_add_use_case_scripts/crowd_journalism/tables/add_event.py 0 → 100644
View file @ 271733ea
from _add_use_case_scripts.requestPost import postLayersToSwagger, postTableToSwagger
def add_table(use_case: str, table_name: str):
'''
take the columns and add the mappings at the server
replace all "/"'s in the internal representation with a "_"
'''
#TODO: split eventEpicenter
#TODO: tags is an array, deal with arrays
columns = [
# "docType",
"objecttype",
"eventid",
#"tags",
"eventEpicenter", #TODO
"range"
]
columns = { c : c for c in columns }
columns["UniqueID"] = "eventid"
columns["firstTag"] = "tags[0]"
table = {
"name": table_name,
"mappings": columns
}
postTableToSwagger(use_case,table)
def add_layers(use_case:str, table_name: str):
layers = [
{ #Useless as all objects are of the same type???
"use_case": use_case,
"table": table_name,
"name": "Object_Type_Layer",
"properties": [
"UniqueID",
"objecttype",
"eventid",
"eventEpicenter",
"range",
"firstTag"
],
"cluster_properties": [
"objecttype"
]
},
{
"use_case": use_case,
"table": table_name,
"name": "Tag_Layer",
"properties": [
"UniqueID",
"objecttype",
"eventid",
"evenEpicenter",
"range",
"firstTag"
],
"cluster_properties": [
"firstTag"
]
},
{
"use_case": use_case,
"table": table_name,
"name": "Event_Epicenter_Layer",
"properties": [
"UniqueID",
"objecttype",
"eventid",
"evenEpicenter",
"range",
"firstTag"
],
"cluster_properties": [
"eventEpicenter"
]
}
]
postLayersToSwagger(use_case,layers)
def main(use_case: str):
print("event")
table_name = "event"
add_table(use_case,table_name)
add_layers(use_case,table_name)
\ No newline at end of file
src/participation-hub/business-logic-microservice/app/_add_use_case_scripts/crowd_journalism/tables/add_purchase.py 0 → 100644
View file @ 271733ea
from _add_use_case_scripts.requestPost import postLayersToSwagger, postTableToSwagger
def add_table(use_case: str, table_name: str):
'''
take the columns and add the mappings at the server
replace all "/"'s in the internal representation with a "_"
'''
columns = [
# "docType",
"objecttype",
"timestamp",
"userid",
"videoid",
"price",
"ownerid"
]
columns = { c : c for c in columns }
columns["UniqueID"] = "userid+videoid+ownerid"
table = {
"name": table_name,
"mappings": columns
}
postTableToSwagger(use_case,table)
def add_layers(use_case:str, table_name: str):
layers = [
{ #Useless as all objects aree Classification?
"use_case": use_case,
"table": table_name,
"name": "Object_Type_Layer",
"properties": [
"UniqueID",
"objecttype",
"timestamp",
"userid",
"videoid",
"price",
"ownerid"
],
"cluster_properties": [
"objecttype"
]
},
{
"use_case": use_case,
"table": table_name,
"name": "Price_Layer",
"properties": [
"UniqueID",
"objecttype",
"timestamp",
"userid",
"videoid",
"price",
"ownerid"
],
"cluster_properties": [
"price"
]
},
{
"use_case": use_case,
"table": table_name,
"name": "Owner_Layer",
"properties": [
"UniqueID",
"objecttype",
"timestamp",
"userid",
"videoid",
"price",
"ownerid"
],
"cluster_properties": [
"ownerid"
]
},
{
"use_case": use_case,
"table": table_name,
"name": "Buyer_Layer",
"properties": [
"UniqueID",
"objecttype",
"timestamp",
"userid",
"videoid",
"price",
"ownerid"
],
"cluster_properties": [
"userid"
]
}
]
postLayersToSwagger(use_case,layers)
def main(use_case: str):
print("purchase")
table_name = "purchase"
add_table(use_case,table_name)
add_layers(use_case,table_name)
\ No newline at end of file
src/participation-hub/business-logic-microservice/app/_add_use_case_scripts/crowd_journalism/tables/add_tag.py 0 → 100644
View file @ 271733ea
from _add_use_case_scripts.requestPost import postLayersToSwagger, postTableToSwagger
def add_table(use_case: str, table_name: str):
'''
take the columns and add the mappings at the server
replace all "/"'s in the internal representation with a "_"
'''
columns = [
# "docType",
"objecttype",
"tag"
]
columns = { c : c for c in columns }
columns["UniqueID"] = "objecttype+tag"
table = {
"name": table_name,
"mappings": columns
}
postTableToSwagger(use_case,table)
def add_layers(use_case:str, table_name: str):
layers = [
{ #Useless as all objects aree the same type??
"use_case": use_case,
"table": table_name,
"name": "Object_Type_Layer",
"properties": [
"UniqueID",
"objecttype",
"tag"
],
"cluster_properties": [
"objecttype"
]
},
{
"use_case": use_case,
"table": table_name,
"name": "Tag_Layer",
"properties": [
"UniqueID",
"objecttype",
"tag"
],
"cluster_properties": [
"tag"
]
}
]
postLayersToSwagger(use_case,layers)
def main(use_case: str):
print("tag")
table_name = "tag"
add_table(use_case,table_name)
add_layers(use_case,table_name)
\ No newline at end of file
src/participation-hub/business-logic-microservice/app/_add_use_case_scripts/crowd_journalism/tables/add_video.py 0 → 100644
View file @ 271733ea
from _add_use_case_scripts.requestPost import postLayersToSwagger, postTableToSwagger
def add_table(use_case: str, table_name: str):
'''
take the columns and add the mappings at the server
replace all "/"'s in the internal representation with a "_"
'''
columns = [
# "docType",
"objecttype",
"videoid",
"duration",
"price",
"creator",
"creationTimestamp",
#"tags",
"geolocation",
"eventid",
"lastupdate",
"md5",
"informativeRating",
"impactRating",
"trustinessRating",
"ready",
"path",
"preview",
#"thumbnails" #not important?
]
columns = { c : c for c in columns }
columns["UniqueID"] = "videoid"
columns["encodedAudio"] = "codec//audio"
columns["encodedVideo"] = "codec//video"
columns["firstTag"] = "tags[0]"
table = {
"name": table_name,
"mappings": columns
}
postTableToSwagger(use_case,table)
def add_layers(use_case:str, table_name: str):
layers = [
{ #Useless as all objects are Classification?
"use_case": use_case,
"table": table_name,
"name": "Object_Type_Layer",
"properties": [
"objecttype",
"videoid",
"duration",
"price",
"creator",
"creationTimestamp",
"lastupdate",
"firstTag"
],
"cluster_properties": [
"objecttype"
]
},
{
"use_case": use_case,
"table": table_name,
"name": "Price_Layer",
"properties": [
"UniqueID",
"objecttype",
"creationTimestamp",
"geolocation",
"videoid",
"price",
"informativeRating",
"impactRating",
"trustinessRating",
"firstTag"
],
"cluster_properties": [
"price"
]
},
{
"use_case": use_case,
"table": table_name,
"name": "Tag_Layer",
"properties": [
"UniqueID",
"objecttype",
"creationTimestamp",
"geolocation",
"videoid",
"price",
"informativeRating",
"impactRating",
"trustinessRating",
"firstTag"
],
"cluster_properties": [
"firstTag"
]
},
{
"use_case": use_case,
"table": table_name,
"name": "Informative_Layer",
"properties": [
"UniqueID",
"objecttype",
"creationTimestamp",
"geolocation",
"videoid",
"price",
"informativeRating",
"impactRating",
"trustinessRating",
"firstTag"
],
"cluster_properties": [
"informativeRating"
]
},
{
"use_case": use_case,
"table": table_name,
"name": "Impact_Layer",
"properties": [
"UniqueID",
"objecttype",
"creationTimestamp",
"geolocation",
"videoid",
"price",
"informativeRating",
"impactRating",
"trustinessRating",
"firstTag"
],
"cluster_properties": [
"impactRating"
]
},
{
"use_case": use_case,
"table": table_name,
"name": "Trust_Layer",
"properties": [
"UniqueID",
"objecttype",
"creationTimestamp",
"geolocation",
"videoid",
"price",
"informativeRating",
"impactRating",
"trustinessRating",
"firstTag"
],
"cluster_properties": [
"trustinessRating"
]
},
{
"use_case": use_case,
"table": table_name,
"name": "Location_Layer",
"properties": [
"UniqueID",
"objecttype",
"creationTimestamp",
"geolocation",
"videoid",
"price",
"informativeRating",
"impactRating",
"trustinessRating",
"firstTag"
],
"cluster_properties": [
"geolocation"
]
},
{
"use_case": use_case,
"table": table_name,
"name": "Video_Age_Layer",
"properties": [
"UniqueID",
"objecttype",
"creationTimestamp",
"geolocation",
"videoid",
"price",
"informativeRating",
"impactRating",
"trustinessRating",
"firstTag"
],
"cluster_properties": [
"creationTimestamp"
]
}
]
postLayersToSwagger(use_case,layers)
def main(use_case: str):
print("video")
table_name = "video"
add_table(use_case,table_name)
add_layers(use_case,table_name)
src/participation-hub/business-logic-microservice/app/_add_use_case_scripts/reddit/add_smart_reddit_schema.py 0 → 100644
View file @ 271733ea
import sys
import os
from pathlib import Path
from typing import Dict, Any
import requests
modules_path = '../../../modules/'
if os.path.exists(modules_path):
sys.path.insert(1, modules_path)
import network_constants as nc
from security.token_manager import TokenManager
import tables.add_reddit as reddit
def add_use_case(use_case: str):
jwt = TokenManager.getInstance().getToken()
url = f"https://articonf1.itec.aau.at:30420/api/use-cases"
response = requests.post(
url,
verify=False,
proxies = { "http":None, "https":None },
headers = { "Authorization": f"Bearer {jwt}"},
json = {"name": use_case}
)
print(url+": "+str(response.status_code))
if __name__ == "__main__":
use_case = "reddit"
# disable ssl warnings :)
requests.packages.urllib3.disable_warnings()
add_use_case(use_case)
reddit.main(use_case)
\ No newline at end of file
src/participation-hub/business-logic-microservice/app/_add_use_case_scripts/reddit/tables/add_reddit.py 0 → 100644
View file @ 271733ea
import network_constants as nc
from security.token_manager import TokenManager
import requests
def add_table(use_case: str, table_name: str):
'''
take the columns and add the mappings at the server
replace all "/"'s in the internal representation with a "_"
'''
columns = {}
use_case = "reddit"
columns = [
# "doctype",
"id",
"user_id",
"title",
"content",
"permalink",
"upvotes",
"percentage_upvoted",
"n_comments",
"subreddit",
"created_at"
]
columns = { c : c for c in columns }
columns["UniqueID"] = "user_id+subreddit+id"
table = {
"name": table_name,
"mappings": columns
}
url = f"https://articonf1.itec.aau.at:30420/api/use-cases/{use_case}/tables"
jwt = TokenManager.getInstance().getToken()
response = requests.post(
url,
verify=False,
proxies = { "http":None, "https":None },
headers = { "Authorization": f"Bearer {jwt}"},
json = table
)
print(url+": "+str(response.status_code))
def add_layers(use_case:str, table_name: str):
layers = [
# { #subreddit is string cannot cluster
# "use_case": use_case,
# "table": table_name,
# "name": "Subreddit_Layer",
# "properties": [
# "UniqueID",
# "subreddit",
# "user_id",
# "title",
# "content",
# "permalink",
# "upvotes",
# "percentage_upvoted",
# "n_comments"
# ],
# "cluster_properties": [
# "subreddit"
# ]
# },
{
"use_case": use_case,
"table": table_name,
"name": "Upvotes_Layer", #TODO Probably do something like Total Votes? so we can get a popularity?
"properties": [
"UniqueID",
"subreddit",
"user_id",
"title",
"content",
"permalink",
"upvotes",
"percentage_upvoted",
"n_comments",
"created_at"
],
"cluster_properties": [
"upvotes",
"percentage_upvoted"
]
},
{
"use_case": use_case,
"table": table_name,
"name": "Percentage_Layer",
"properties": [
"UniqueID",
"subreddit",
"user_id",
"title",
"content",
"permalink",
"upvotes",
"percentage_upvoted",
"n_comments",
"created_at"
],
"cluster_properties": [
"percentage_upvoted"
]
},
{
"use_case": use_case,
"table": table_name,
"name": "Engagement_Layer",
"properties": [
"UniqueID",
"subreddit",
"user_id",
"title",
"content",
"permalink",
"upvotes",
"percentage_upvoted",
"n_comments",
"created_at"
],
"cluster_properties": [
"n_comments"
]
},
{
"use_case": use_case,
"table": table_name,
"name": "Time_Layer",
"properties": [
"UniqueID",
"subreddit",
"user_id",
"title",
"content",
"permalink",
"upvotes",
"percentage_upvoted",
"n_comments",
"created_at"
],
"cluster_properties": [
"created_at"
]
}
]
jwt = TokenManager.getInstance().getToken()
for layer in layers:
url = f"https://articonf1.itec.aau.at:30420/api/layers"
response = requests.post(
url,
verify=False,
proxies = { "http":None, "https":None },
headers = { "Authorization": f"Bearer {jwt}"},
json = layer
)
print(url+": "+str(response.status_code))
def main(use_case: str):
print("reddit")
table_name = "reddit"
add_table(use_case,table_name)
add_layers(use_case,table_name)
src/participation-hub/business-logic-microservice/app/_add_use_case_scripts/car_sharing/tables/requestPost.py → src/participation-hub/business-logic-microservice/app/_add_use_case_scripts/requestPost.py
View file @ 271733ea
......@@ -2,6 +2,7 @@
import network_constants as nc
from security.token_manager import TokenManager
import requests
from typing import List
def postTableToSwagger(use_case:str, table:dict ):
......@@ -20,7 +21,7 @@ def postTableToSwagger(use_case:str, table:dict ):
print(url+": "+str(response.status_code)+" MSG:"+str(response.content))
def postLayersToSwagger(use_case:str, layers:dict):
def postLayersToSwagger(use_case:str, layers: List[dict]):
jwt = TokenManager.getInstance().getToken()
......
src/participation-hub/business-logic-microservice/app/_add_use_case_scripts/vialog/add_vialog_schema.py 0 → 100644
View file @ 271733ea
import sys
import os
from pathlib import Path
from typing import Dict, Any
import requests
modules_paths = ['.', '../../../modules/']
for modules_path in modules_paths:
if os.path.exists(modules_path):
sys.path.insert(1, modules_path)
from _add_use_case_scripts.vialog.tables import add_user, add_video, add_change
import network_constants as nc
from security.token_manager import TokenManager
def add_use_case(use_case: str):
#use_case = "vialog"
jwt = TokenManager.getInstance().getToken()
url = f"https://articonf1.itec.aau.at:30420/api/use-cases"
response = requests.post(
url,
verify=False,
proxies = { "http":None, "https":None },
headers = { "Authorization": f"Bearer {jwt}"},
json = {"name": use_case}
)
print(url+": "+str(response.content))
if __name__ == "__main__":
use_case = "vialog"
# disable ssl warnings :)
requests.packages.urllib3.disable_warnings()
add_use_case(use_case)
add_user.main(use_case)
add_video.main(use_case)
add_change.main(use_case)
\ No newline at end of file
src/participation-hub/business-logic-microservice/app/_add_use_case_scripts/vialog/add_vialog_schema_new_enum.py 0 → 100644
View file @ 271733ea
import sys
import os
from pathlib import Path
from typing import Dict, Any
import requests
modules_paths = ['.', '../../../modules/']
for modules_path in modules_paths:
if os.path.exists(modules_path):
sys.path.insert(1, modules_path)
from _add_use_case_scripts.vialog.tables import add_user, add_video, add_change
import network_constants as nc
from security.token_manager import TokenManager
def add_use_case(use_case: str):
#use_case = "vialog"
jwt = TokenManager.getInstance().getToken()
url = f"https://articonf1.itec.aau.at:30420/api/use-cases"
response = requests.post(
url,
verify=False,
proxies = { "http":None, "https":None },
headers = { "Authorization": f"Bearer {jwt}"},
json = {"name": use_case}
)
print(url+": "+str(response.content))
if __name__ == "__main__":
use_case = "vialog-new-enum"
# disable ssl warnings :)
requests.packages.urllib3.disable_warnings()
add_use_case(use_case)
add_video.main(use_case)
add_change.main(use_case)
\ No newline at end of file
src/participation-hub/business-logic-microservice/app/_add_use_case_scripts/vialog/tables/add_change.py 0 → 100644
View file @ 271733ea
from _add_use_case_scripts.requestPost import postLayersToSwagger, postTableToSwagger
def add_table(use_case: str, table_name: str):
'''
take the columns and add the mappings at the server
replace all "/"'s in the internal representation with a "_"
'''
columns = [
"changeType",
"changedValue",
"previousValue",
"newValue",
]
columns = { c : c for c in columns }
columns["UniqueID"] = "videoId+changeId"
columns["changeType"] = "enum(changeType)"
columns["changedValue"] = "enum(changedValue)"
table = {
"name": table_name,
"mappings": columns
}
postTableToSwagger(use_case,table)
def add_layers(use_case:str, table_name: str):
layers = [
{
"use_case": use_case,
"table": table_name,
"name": "Changetype_Layer",
"properties": [
"changeType",
],
"cluster_properties": [
"changeType",
]
},
{
"use_case": use_case,
"table": table_name,
"name": "Changedvalue_Layer",
"properties": [
"changedValue",
"previousValue",
"newValue"
],
"cluster_properties": [
"changedValue"
]
}
]
postLayersToSwagger(use_case,layers)
def main(use_case: str):
print("user")
table_name = "change"
add_table(use_case,table_name)
add_layers(use_case,table_name)
\ No newline at end of file
src/participation-hub/business-logic-microservice/app/_add_use_case_scripts/vialog/tables/add_user.py 0 → 100644
View file @ 271733ea
from _add_use_case_scripts.requestPost import postLayersToSwagger, postTableToSwagger
def add_table(use_case: str, table_name: str):
'''
take the columns and add the mappings at the server
replace all "/"'s in the internal representation with a "_"
'''
columns = [
# "docType",
"userId",
"rewardBalance"
]
columns = { c : c for c in columns }
columns["UniqueID"] = "userId"
table = {
"name": table_name,
"mappings": columns
}
postTableToSwagger(use_case,table)
def add_layers(use_case:str, table_name: str):
layers = [
{
"use_case": use_case,
"table": table_name,
"name": "User_Layer",
"properties": [
"UniqueID",
"rewardBalance"
],
"cluster_properties": [
"UniqueID",
]
},
{
"use_case": use_case,
"table": table_name,
"name": "User_Balance_Layer",
"properties": [
"UniqueID",
"rewardBalance"
],
"cluster_properties": [
"rewardBalance"
]
}
]
postLayersToSwagger(use_case,layers)
def main(use_case: str):
print("user")
table_name = "user"
add_table(use_case,table_name)
add_layers(use_case,table_name)
\ No newline at end of file
src/participation-hub/business-logic-microservice/app/_add_use_case_scripts/vialog/tables/add_video.py 0 → 100644
View file @ 271733ea
from _add_use_case_scripts.requestPost import postLayersToSwagger, postTableToSwagger
def add_table(use_case: str, table_name: str):
'''
take the columns and add the mappings at the server
replace all "/"'s in the internal representation with a "_"
'''
columns = [
'videoId',
'objType',
'eventName',
'video_token',
'replyTo',
'created',
'duration',
'videoResolution',
'label',
'threadId',
'position',
'views',
'moderatedBy',
'moderationDate',
'communityManagerNotes',
'rewards',
'video_state',
'video_type'
]
columns = { c : c for c in columns }
columns["UniqueID"] = "videoId"
columns["moderatedBy"] = "enum(moderatedBy)"
columns["video_state"] = "enum(video_state)"
columns["video_type"] = "enum(video_type)"
columns["videoResolution"] = "enum(videoResolution)"
table = {
"name": table_name,
"mappings": columns
}
postTableToSwagger(use_case,table)
def add_layers(use_case:str, table_name: str):
layers = [
{
"use_case": use_case,
"table": table_name,
"name": "Manager_Layer",
"properties": [
"UniqueID",
"moderationDate",
"moderatedBy",
"video_state",
"video_type"
],
"cluster_properties": [
"moderatedBy",
"video_state"
]
},
{
"use_case": use_case,
"table": table_name,
"name": "Video_Popularity_Layer",
"properties": [
"UniqueID",
"label",
"created",
"views",
"rewards",
"video_state",
"video_type"
],
"cluster_properties": [
"views",
"video_type"
]
},
{
"use_case": use_case,
"table": table_name,
"name": "Video_Age_Layer",
"properties": [
"UniqueID",
"label",
"created",
"views",
"rewards",
"video_state",
"video_type"
],
"cluster_properties": [
"created"
]
},
{
"use_case": use_case,
"table": table_name,
"name": "Rewards_Layer",
"properties": [
"UniqueID",
"label",
"created",
"views",
"rewards",
"video_state",
"video_type"
],
"cluster_properties": [
"rewards"
]
},
{
"use_case": use_case,
"table": table_name,
"name": "Video_Lenght_Layer",
"properties": [
"UniqueID",
"created",
"views",
"duration",
"video_state",
"video_type"
],
"cluster_properties": [
"duration"
]
},
{
"use_case": use_case,
"table": table_name,
"name": "Video_Resolution_Layer",
"properties": [
"UniqueID",
"created",
"views",
"videoResolution",
"video_state",
"video_type"
],
"cluster_properties": [
"videoResolution"
]
}
]
postLayersToSwagger(use_case,layers)
def main(use_case: str):
print("Video")
table_name = "video"
add_table(use_case,table_name)
add_layers(use_case,table_name)
\ No newline at end of file
src/participation-hub/business-logic-microservice/app/requirements.txt
View file @ 271733ea
......@@ -8,6 +8,7 @@ click==7.1.2
clickclick==1.2.2
colorama==0.4.3
connexion==2.7.0
coverage==5.3.1
cryptography==3.1
Flask==1.1.2
idna==2.10
......
src/participation-hub/business-logic-microservice/app/test_coverage.py 0 → 100644
View file @ 271733ea
import unittest
import sys
for path in ['../', './']:
sys.path.insert(1, path)
#####################################
### Don't include for test report ###
#####################################
try:
class TestCoverage(unittest.TestCase):
def test_init_main(self):
try:
# python -m unittest discover
# add modules folder to interpreter path
import sys
import os
from pathlib import Path
from typing import Dict, Any
modules_path = '../../../modules/'
if os.path.exists(modules_path):
sys.path.insert(1, modules_path)
# load swagger config
import connexion
from security import swagger_util
from env_info import is_running_locally, get_resources_path
from flask import request
from flask import redirect
app = connexion.App(__name__, specification_dir='configs/')
from db.entities.layer_adapter import LayerAdapter
except Exception as e:
print ("Exception found:")
print (e)
try:
import main
except Exception as e:
print ("Exception found:")
print (e)
def test_db_main(self):
try:
import network_constants as netconst
from database.MongoRepositoryBase import MongoRepositoryBase
from db.entities import layer_adapter
from db.entities import table
from db.entities import use_case
import pymongo
import json
from typing import List, Dict
except Exception as e:
print ("Exception found:")
print (e)
try:
from db import repository
from db import table_repository
from db import use_case_repository
except Exception as e:
print ("Exception found:")
print (e)
def test_routes(self):
try:
from routes import layer
except Exception as e:
print ("Exception found:")
print (e)
try:
from routes import tables
except Exception as e:
print ("Exception found:")
print (e)
try:
from routes import use_case
except Exception as e:
print ("Exception found:")
print (e)
def test_services(self):
try:
from services import layer_adapter_service
except Exception as e:
print ("Exception found:")
print (e)
def test_use_case_scripts(self):
try:
import network_constants as nc
from security.token_manager import TokenManager
import requests
from typing import List
from _add_use_case_scripts import requestPost
except Exception as e:
print ("Exception found:")
print (e)
#######
#from _add_use_case_scripts.bank-app import add_bank_app_schema ##eror not importing? invalid folder name?
#from _add_use_case_scripts.bank-app.tables import add_bank_app_schema
try:
from _add_use_case_scripts.car_sharing import add_carsharing_schema
from _add_use_case_scripts.car_sharing.tables import add_car
from _add_use_case_scripts.car_sharing.tables import add_hash
from _add_use_case_scripts.car_sharing.tables import add_offer
from _add_use_case_scripts.car_sharing.tables import add_publication
from _add_use_case_scripts.car_sharing.tables import add_travel
from _add_use_case_scripts.car_sharing.tables import add_user
except Exception as e:
print ("Exception found:")
print (e)
try:
from _add_use_case_scripts.crowd_journalism import add_crowdjournalism_schema
from _add_use_case_scripts.crowd_journalism.tables import add_classification
from _add_use_case_scripts.crowd_journalism.tables import add_event
from _add_use_case_scripts.crowd_journalism.tables import add_purchase
from _add_use_case_scripts.crowd_journalism.tables import add_tag
from _add_use_case_scripts.crowd_journalism.tables import add_video
except Exception as e:
print ("Exception found:")
print (e)
try:
from _add_use_case_scripts.debug import add_debug_schema
from _add_use_case_scripts.debug.tables import add_pizza_table
except Exception as e:
print ("Exception found:")
print (e)
#from _add_use_case_scripts.smart-energy import add_smart_energy_schema
#from _add_use_case_scripts.smart-energy.tables import add_smart_energy
try:
from _add_use_case_scripts.vialog import add_vialog_schema
from _add_use_case_scripts.vialog.tables import add_user
from _add_use_case_scripts.vialog.tables import add_video
except Exception as e:
print ("Exception found:")
print (e)
if __name__ == '__main__':
unittest.main()
except Exception as e:
print ("Exception found:")
print (e)
\ No newline at end of file
src/participation-hub/business-logic-microservice/app/tests/test_layer_adapter.py
View file @ 271733ea
......@@ -4,7 +4,7 @@ from db.entities.layer_adapter import LayerAdapter
class Test_Layer_Adapter(unittest.TestCase):
def test_valid_adapter(self):
def test_LayerAdapter_newLayerAdapterObj_validInstantiation(self):
adapter1 = LayerAdapter("layer1", "use_case", "table", ["a", "c"], ["a"])
print(adapter1.to_serializable_dict)
......
src/rest-gateway/app/main.py
View file @ 271733ea
......@@ -21,33 +21,37 @@ import connexion
from security import swagger_util
from env_info import is_running_locally, get_resources_path
from flask import request
from flask_cors import CORS
from flask import redirect
app = connexion.App(__name__, specification_dir='configs/')
@app.app.before_request
def before_request():
# start app
if __name__ == '__main__':
app = connexion.App(__name__, specification_dir='configs/')
CORS(app.app)
@app.app.before_request
def before_request():
if request.url.startswith('http://'):
url = request.url.replace('http://', 'https://', 1)
code = 301
return redirect(url, code=code)
@app.route('/', methods=['GET'])
def api_root():
@app.route('/', methods=['GET'])
def api_root():
return redirect('/api/ui')
# SSL configuration
certificate_path = get_resources_path()
context = (os.path.normpath(f'{certificate_path}/articonf1.crt'), os.path.normpath(f'{certificate_path}/articonf1.key')) # certificate and key files
# SSL configuration
certificate_path = get_resources_path()
context = (os.path.normpath(f'{certificate_path}/articonf1.crt'), os.path.normpath(f'{certificate_path}/articonf1.key')) # certificate and key files
if is_running_locally():
if is_running_locally():
print("Running locally...")
app.add_api(swagger_util.get_bundled_specs(Path("configs/swagger_local.yml")),
resolver = connexion.RestyResolver("cms_rest_api"))
else:
else:
app.add_api(swagger_util.get_bundled_specs(Path("configs/swagger.yml")),
resolver = connexion.RestyResolver("cms_rest_api"))
# start app
if __name__ == '__main__':
app.run(host='0.0.0.0', port=5000, debug=True, use_reloader=False, ssl_context=context) # disable reloader so only subscribed once to rabbitmqö
\ No newline at end of file
src/rest-gateway/app/pull_video_data.py 0 → 100644
View file @ 271733ea
import requests
import json
videoListUrl = "https://dev758755.vialog.app/Videos/Meta/ListAll"
videoUrl = "https://dev758755.vialog.app/stat/events?type=video&id="
# token from Rest Gateway to authorize
JWT_TOKEN = 'eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJ1c2VybmFtZSI6InJlZ3VsYXJAaXRlYy5hYXUuYXQiLCJjcmVhdGVkX2F0IjoiMjAyMS0wNC0wNyAxMjo0OTo0MS43MTkzNjQiLCJ2YWxpZF91bnRpbCI6IjIwMjEtMDQtMDggMTI6NDk6NDEuNzE5MzY0In0.FN6qqBQeJSmXtS0-0dBiL-ojz6Ou7E5Tc9macrrhM4A'
useCase = "vialog-new-enum"
def send_transaction_to_rest_gateway(transaction: dict):
res = requests.post(
url = 'https://articonf1.itec.aau.at:30401/api/trace',
json = transaction,
headers = {"Authorization": f"Bearer {JWT_TOKEN}"},
verify = False # ignore ssl error
)
print(res)
videosRequest = requests.get(videoListUrl)
empty = set()
html = set()
if (videosRequest.status_code != 200):
print(f"Status: {videosRequest.status_code}")
dataCount = 0
for video in videosRequest.json():
dataCount += 1
id = video["videoId"]
videoRequest = requests.get(f"{videoUrl}{id}")
if videoRequest.status_code != 200:
print(f"Status: {videoRequest.status_code}")
if videoRequest.text != "" and not videoRequest.text.startswith("<!DOCTYPE html>"):
print(f"\n\n{videoUrl}{id}")
historyList = sorted(videoRequest.json()[0]["History"], key=lambda k: k['moderationDate'], reverse=True)
historyList.append(empty)
i = 0
changeList = []
for eventMap in historyList:
if historyList[i+1] == empty:
break
if i == 0:
lastState = eventMap
lastState["ApplicationType"] = useCase
lastState["docType"] = "video"
print(f"{lastState}\n\n\n")
send_transaction_to_rest_gateway(lastState)
if eventMap["eventName"] == 'r1eabcbdc8f5378b2ba71a1b6fe2038b Created' or eventMap["eventName"] == 'Created':
change = {"changeType": "Created", "changedValue": "video_state", "previousValue": "", "newValue":"Created"}
elif eventMap["eventName"] == "Restore":
change = {"changeType": "Restore", "changedValue": "", "previousValue": "", "newValue":""}
elif eventMap["eventName"] == "PositionChange":
change = {"changeType": "PositionChange", "changedValue": "position", "previousValue": historyList[i+1]["position"], "newValue": eventMap["position"]}
elif eventMap["eventName"] == "Hide":
change = {"changeType": "Hide", "changedValue": "video_state", "previousValue": historyList[i+1]["video_state"], "newValue": eventMap["video_state"]}
elif eventMap["eventName"] == "Publish":
change = {"changeType": "Publish", "changedValue": "video_state", "previousValue": historyList[i+1]["video_state"], "newValue":eventMap["video_state"]}
elif eventMap["eventName"] == "CMNote":
change = {"changeType": "CMNote", "changedValue": "communityManagerNotes", "previousValue": historyList[i+1]["communityManagerNotes"], "newValue":eventMap["communityManagerNotes"]}
elif eventMap["eventName"] == "Move":
change = {"changeType": "Move", "changedValue": "position", "previousValue": historyList[i+1]["position"], "newValue":eventMap["position"]}
elif eventMap["eventName"] == "VideoType":
change = {"changeType": "VideoType", "changedValue": "video_type", "previousValue": historyList[i+1]["video_type"], "newValue":eventMap["video_type"]}
elif eventMap["eventName"] == "Delete":
change = {"changeType": "Delete", "changedValue": "", "previousValue": "", "newValue": ""}
elif eventMap["eventName"] == "Copy":
change = {"changeType": "Copy", "changedValue": "", "previousValue": "", "newValue":""}
elif eventMap["eventName"] == "CustomLabel":
change = {"changeType": "CustomLabel", "changedValue": "label", "previousValue": historyList[i+1]["label"], "newValue":eventMap["label"]}
change["videoId"] = id
change["changeId"] = i
change["timestamp"] = eventMap["moderationDate"]
change["ApplicationType"] = useCase
change["docType"] = "change"
print(change)
send_transaction_to_rest_gateway(change)
i += 1
elif videoRequest.text == "":
empty.add(id)
else:
html.add(id)
print(f"empty: {empty}\n\n")
print(f"html page: {html}\n\n")
print(f"history: {historyList}")
print(dataCount)
src/rest-gateway/app/requirements.txt
View file @ 271733ea
......@@ -12,6 +12,7 @@ connexion==2.7.0
cryptography==2.9.2
Deprecated==1.2.10
Flask==1.1.2
Flask-Cors==3.0.10
idna==2.10
importlib-metadata==1.7.0
inflection==0.5.0
......
src/rest-gateway/app/routes/blockchain_trace.py
View file @ 271733ea
......@@ -9,7 +9,7 @@ def receive():
body = request.json
if isBlockchainTraceValid(body):
message = {'type': 'blockchain-transaction', 'content': body}
message = {'type': 'blockchain-transaction', 'retries': 0, 'content': body}
message_sender.send_message('inhub', 'trace-retrieval', json.dumps(message))
return Response(status=201)
......
src/rest-gateway/app/test_coverage.py 0 → 100644
View file @ 271733ea
import unittest
import sys
for path in ['../', './']:
sys.path.insert(1, path)
#####################################
### Don't include for test report ###
#####################################
try:
class TestCoverage(unittest.TestCase):
def test_init_main(self):
try:
print("Entered test main")
# python -m unittest discover
# add modules folder to interpreter path
import sys
import os
import prance
from pathlib import Path
from typing import Dict, Any
modules_path = '../../modules/'
if os.path.exists(modules_path):
sys.path.insert(1, modules_path)
# init logging to file
import logging
LOG_FORMAT = ('%(levelname) -5s %(asctime)s %(name)s:%(funcName) -35s %(lineno) -5d: %(message)s')
logging.basicConfig(level=logging.INFO, format=LOG_FORMAT)
LOGGER = logging.getLogger(__name__)
except Exception as e:
print ("Exception found:")
print (e)
#################################
try:
import connexion
from security import swagger_util
from env_info import is_running_locally, get_resources_path
from flask import request
from flask import redirect
except Exception as e:
print ("Exception found:")
print (e)
try:
import main #something in main is causing an infinite loop (probably an async task/ listener)
except Exception as e:
print ("Exception found:")
print (e)
print("Finished test main")
def test_database(self):
try:
print("Entered test db")
import network_constants as netconst
from database.MongoRepositoryBase import MongoRepositoryBase
import pymongo
import json
from db.entities.user import User
from typing import List
except Exception as e:
print ("Exception found:")
print (e)
try:
from db import repository
except Exception as e:
print ("Exception found:")
print (e)
print("Finished test db")
def test_services(self):
try:
print("Entered test services")
from functools import wraps
from flask import g, request, redirect, url_for
# global imports (dont't worry, red is normal)
from db.repository import Repository
from db.entities.user import User
from services.user_service import UserService
from env_info import get_resources_path
import jwt
from datetime import datetime, timedelta
from typing import Dict
import bcrypt
except Exception as e:
print ("Exception found:")
print (e)
try:
from services import login_wrapper
except Exception as e:
print ("Exception found:")
print (e)
try:
from services import token_service
except Exception as e:
print ("Exception found:")
print (e)
try:
from services import user_service
except Exception as e:
print ("Exception found:")
print (e)
print("Finished test services")
def test_routes(self):
try:
print("Entered test routes")
from flask import request, Response
from messaging.ReconnectingMessageManager import ReconnectingMessageManager
import json
from flask import request
# global imports (dont't worry, red is normal)
from db.entities.user import User
from services.user_service import UserService
from services.login_wrapper import login_required
from services.token_service import TokenService
import bcrypt
import jwt
except Exception as e:
print ("Exception found:")
print (e)
try:
from routes import user
except Exception as e:
print ("Exception found:")
print (e)
#from routes import blockchain_trace #message_sender in blockchain_trace is causing an infinite loop (probabily an async task//listener)
try:
from routes import debug
except Exception as e:
print ("Exception found:")
print (e)
print("Finished test routes")
def test_add_users(self):
try:
print("Entered test users")
# add modules folder to interpreter path
import sys
import os
import json
import prance
from pathlib import Path
from typing import Dict, Any
modules_path = '../../modules/'
if os.path.exists(modules_path):
sys.path.insert(1, modules_path)
from services.user_service import UserService
from env_info import is_running_locally, get_resources_path
except Exception as e:
print ("Exception found:")
print (e)
try:
import add_users
except Exception as e:
print ("Exception found:")
print (e)
print("Finished test users")
if __name__ == '__main__':
unittest.main()
except Exception as e:
print ("Exception found:")
print (e)
\ No newline at end of file
src/transaction-hub-in/trace-retrieval-microservice/app/configs/routes.yml
View file @ 271733ea
......@@ -62,6 +62,39 @@ paths:
responses:
'200':
description: "Successful Request"
/use_cases/{use_case}/transactions-duplicated:
delete:
security:
- JwtRegular: []
operationId: "routes.transactions.delete_all_duplicated_for_use_case"
tags:
- "Transactions"
summary: "Deletes all duplicated Transactions in the given Use-Case"
description: "Deletes all duplicated Transactions in the given Use-Case"
parameters:
- in: path
name: "use_case"
required: true
type: "string"
responses:
'200':
description: "Successful Request"
get:
security:
- JwtRegular: []
operationId: "routes.transactions.all_duplicated_for_use_case"
tags:
- "Transactions"
summary: "Retrieves all duplicated Transactions in the given Use-Case"
description: "Retrieves all duplicated Transactions in the given Use-Case"
parameters:
- in: path
name: "use_case"
required: true
type: "string"
responses:
'200':
description: "Successful Request"
/debug:
post:
......
src/transaction-hub-in/trace-retrieval-microservice/app/database/repository.py
View file @ 271733ea
......@@ -18,26 +18,35 @@ class Repository(MongoRepositoryBase):
self._transaction_collection = 'transactions'
self._failed_transaction_collection = 'transactions_failed'
self._duplicated_transaction_collection = "transactions_duplicated"
def delete_all_transactions(self):
collection = self._database[self._transaction_collection]
collection.delete_many({})
def add_transaction(self, transaction: Transaction):
reference = self.get_transaction_with_id(transaction.id())
'''
Adds a transaction to mongodb repository.
@throws
KeyError - Duplicate transaction ID
'''
use_case= transaction.use_case
table = transaction.table
reference = self.get_transaction_with_id(transaction.id(),use_case, table)
if reference == None:
super().insert_entry(self._transaction_collection, transaction.to_serializable_dict())
else:
raise ValueError(f"ID {transaction['UniqueID']} already exists")
raise KeyError(f"ID {transaction['UniqueID']} already exists")
def all_transactions_for_use_case(self, use_case: str) -> List[Transaction]:
result = super().get_entries(self._transaction_collection, projection={'_id': False}, selection={"use_case": use_case})
return [Transaction.from_serializable_dict(row) for row in list(result)]
def get_transaction_with_id(self, unique_id: str) -> Transaction:
result = list(super().get_entries(self._transaction_collection, projection={'_id': False}, selection={"UniqueID": unique_id}))
def get_transaction_with_id(self, unique_id: str, use_case:str, table:str ) -> Transaction:
result = list(super().get_entries(self._transaction_collection, projection={'_id': False}, selection={"id": unique_id,"use_case": use_case, "table":table}))
if len(result) == 1:
if len(result) >= 1:
return Transaction.from_serializable_dict(result[0])
return None
......@@ -58,3 +67,15 @@ class Repository(MongoRepositoryBase):
def delete_all_failed_transactions(self, use_case:str):
collection = self._database[self._failed_transaction_collection]
collection.delete_many({"ApplicationType": use_case})
def add_duplicated_transaction(self, transaction: Transaction):
#transaction["timestamp"] = time.time()
super().insert_entry(self._duplicated_transaction_collection, transaction.to_serializable_dict())
def all_duplicated_transactions_for_use_case(self, use_case: str) -> List[Dict]:
result = super().get_entries(self._duplicated_transaction_collection, projection={'_id': False}, selection={"use_case": use_case})
return [Transaction.from_serializable_dict(row) for row in list(result)]
def delete_all_duplicated_transactions(self, use_case:str):
collection = self._database[self._duplicated_transaction_collection]
collection.delete_many({"ApplicationType": use_case})
\ No newline at end of file
src/transaction-hub-in/trace-retrieval-microservice/app/main.py
View file @ 271733ea
......@@ -27,7 +27,7 @@ from flask import request
from flask import redirect
# init message handler
message_handler = MessageHandler(Repository(), ReconnectingMessageManager.getInstance(), RestFetcher())
message_handler:MessageHandler = None
def message_received_callback(channel, method, properties, body):
message_handler.handle_generic(body)
......@@ -59,6 +59,7 @@ else:
# start app
if __name__ == '__main__':
message_handler = MessageHandler(Repository(), ReconnectingMessageManager.getInstance(), RestFetcher())
message_manager = ReconnectingMessageManager.getInstance()
message_manager.start_consuming('inhub', 'direct', 'trace-retrieval', True, message_received_callback)
......
src/transaction-hub-in/trace-retrieval-microservice/app/messaging/MessageHandler.py
View file @ 271733ea
......@@ -8,9 +8,11 @@ import json
import hashlib
import logging
import requests
requests.packages.urllib3.disable_warnings()
from typing import Dict
LOGGER = logging.getLogger(__name__)
class MessageHandler:
......@@ -30,7 +32,7 @@ class MessageHandler:
self._rest_fetcher = rest_fetcher
def handle_generic(self, body):
LOGGER.info(f"Received message: {body}")
result = None
message = None
......@@ -39,11 +41,13 @@ class MessageHandler:
except (ValueError, TypeError):
result = self.MSG_NOT_JSON
LOGGER.warning(result)
LOGGER.info(f"Received message: {body}")
return result
if not 'type' in message:
result = self.MSG_NO_TYPE
LOGGER.warning(result)
LOGGER.info(f"Received message: {body}")
return result
if message['type'] == 'blockchain-transaction':
......@@ -51,8 +55,11 @@ class MessageHandler:
result = self.MSG_TRACE_PROCESSED
else:
result = self.MSG_NOT_PROCESSED
LOGGER.warning(result)
LOGGER.info(f"Received message: {body}")
LOGGER.info(result)
#LOGGER.info(result) #too much spam
return result
def _resolve_path(self, data: Dict, path:str) -> Dict:
......@@ -132,56 +139,74 @@ class MessageHandler:
'''
# check if there is a use-case in the message
if "ApplicationType" not in transaction_message.keys():
if "ApplicationType" not in transaction_message:
LOGGER.error("Transaction has no ApplicationType, storing it under use-case 'unknown'.")
transaction_message["ApplicationType"] = "unknown"
self._mongo_repo.add_failed_transaction(transaction_message)
return
# check if there is a doctype in the message
if "docType" not in transaction_message.keys():
LOGGER.error("Transaction has no docType, ignoring it.")
# check if there is a table in the message
if "docType" not in transaction_message:
LOGGER.error("Transaction has no docType, storing it under table 'unknown'.")
transaction_message["docType"] = "unknown"
self._mongo_repo.add_failed_transaction(transaction_message)
return
use_case = transaction_message["ApplicationType"]
docType = transaction_message["docType"]
transaction_use_case = transaction_message["ApplicationType"]
transaction_table = transaction_message["docType"]
try:
tables = self._rest_fetcher.fetch_schema_information(use_case)
tables = self._rest_fetcher.fetch_schema_information(transaction_use_case)
except ValueError as e:
print(f"{e}")
LOGGER.error(f"{e}\nCould not fetch schema, storing it as a failed transaction..")
self._mongo_repo.add_failed_transaction(transaction_message)
return
target_table = None
# find correct table
for table in tables:
if table["name"] == docType:
if table["name"] == transaction_table:
target_table = table
break
# abort if table does not exist.
if target_table == None:
LOGGER.error(f"There is no table '{docType}', ignoring the message.")
LOGGER.error(f"There is no table '{transaction_table}', storing it as a failed transaction.")
self._mongo_repo.add_failed_transaction(transaction_message)
return
mappings = table["mappings"]
try:
flattened = self._flatten_transaction(transaction_message, mappings)
except KeyError as e:
LOGGER.error(f"Failed while flattening with KeyError: {str(e)}")
LOGGER.error(f"Failed while flattening with KeyError: {str(e)}, storing it as a failed transaction.")
self._mongo_repo.add_failed_transaction(transaction_message)
return
transaction = Transaction(use_case, target_table["name"], flattened)
transaction = Transaction(transaction_use_case, target_table["name"], flattened)
#check for duplicates
try:
self._mongo_repo.add_transaction(transaction)
references = self._mongo_repo.get_transaction_with_id(transaction.id(),transaction_use_case,transaction_table)
if references != None:
LOGGER.info("Found duplicate, storing it as a duplicated transaction.")
self._mongo_repo.add_duplicated_transaction(transaction)
return
except ValueError as e:
LOGGER.error(f"{e}, ignoring node")
LOGGER.error(f"{e}, could not insert duplicated node.")
return
try:
self._mongo_repo.add_transaction(transaction)
except KeyError as e:
LOGGER.error(f"{e}, ignored {transaction_message}")
# self._mongo_repo.add_failed_transaction(transaction_message)
return
msg = {
"type": "new-trace",
"content": transaction.to_serializable_dict(),
"content": transaction.to_serializable_dict()
}
msg_string = json.dumps(msg)
......
src/transaction-hub-in/trace-retrieval-microservice/app/messaging/dummy_rest_fetcher.py
View file @ 271733ea
......@@ -2,7 +2,10 @@ from messaging.rest_fetcher import RestFetcher
class DummyRestFetcher(RestFetcher):
def fetch_schema_information(self, use_case: str):
return [
returnList = []
if use_case == "string":
returnList =[
{
"name": "string",
"use_case": "string",
......@@ -10,5 +13,33 @@ class DummyRestFetcher(RestFetcher):
"UniqueID": "ResourceIds",
"RIds": "ResourceIds"
}
},
{
"name": "string2",
"use_case": "string",
"mappings": {
"UniqueID": "ResourceIds",
"RIds": "ResourceIds"
}
}
]
else:
returnList = [
{
"name": "string",
"use_case": "string2",
"mappings": {
"UniqueID": "ResourceIds",
"RIds": "ResourceIds"
}
},
{
"name": "string2",
"use_case": "string2",
"mappings": {
"UniqueID": "ResourceIds",
"RIds": "ResourceIds"
}
}
]
return returnList
\ No newline at end of file
src/transaction-hub-in/trace-retrieval-microservice/app/messaging/rest_fetcher.py
View file @ 271733ea
......@@ -18,7 +18,7 @@ class RestFetcher:
)
if response.status_code != 200:
raise ValueError("Error while retrieving schema information")
raise ValueError(f"Error while retrieving schema information.\tStatus-code:{response.status_code}")
tables = json.loads(response.text)
return tables
src/transaction-hub-in/trace-retrieval-microservice/app/requirements.txt
View file @ 271733ea
......@@ -7,6 +7,7 @@ click==7.1.2
clickclick==1.2.2
colorama==0.4.3
connexion==2.7.0
coverage==5.3.1
cryptography==3.1
Deprecated==1.2.10
Flask==1.1.2
......
src/transaction-hub-in/trace-retrieval-microservice/app/routes/transactions.py
View file @ 271733ea
......@@ -21,3 +21,11 @@ def all_failed_for_use_case(use_case: str):
def delete_all_failed_for_use_case(use_case: str):
_repository.delete_all_failed_transactions(use_case)
return Response(status=200)
def all_duplicated_for_use_case(use_case: str):
transactions = _repository.all_duplicated_transactions_for_use_case(use_case)
return [t.to_serializable_dict() for t in transactions]
def delete_all_duplicated_for_use_case(use_case: str):
_repository.delete_all_duplicated_transactions(use_case)
return Response(status=200)
\ No newline at end of file
src/transaction-hub-in/trace-retrieval-microservice/app/test_coverage.py 0 → 100644
View file @ 271733ea
import unittest
import sys
for path in ['../', './']:
sys.path.insert(1, path)
#####################################
### Don't include for test report ###
#####################################
try:
class TestCoverage(unittest.TestCase):
def test_init_main(self):
try:
# python -m unittest discover
# add modules folder to interpreter path
import sys
import os
import prance
from pathlib import Path
modules_path = '../../../modules/'
if os.path.exists(modules_path):
sys.path.insert(1, modules_path)
# init logging to file
import logging
LOG_FORMAT = ('%(levelname) -5s %(asctime)s %(name)s:%(funcName) -35s %(lineno) -5d: %(message)s')
logging.basicConfig(level=logging.INFO, format=LOG_FORMAT)
LOGGER = logging.getLogger(__name__)
except Exception as e:
print ("Exception found:")
print (e)
#############################
try:
import connexion
from security import swagger_util
from env_info import is_running_locally, get_resources_path
except Exception as e:
print ("Exception found:")
print (e)
try:
from database.repository import Repository
except Exception as e:
print ("Exception found:")
print (e)
try:
from messaging.MessageHandler import MessageHandler
except Exception as e:
print ("Exception found:")
print (e)
try:
from messaging.ReconnectingMessageManager import ReconnectingMessageManager
except Exception as e:
print ("Exception found:")
print (e)
try:
from messaging.rest_fetcher import RestFetcher
from flask import request
from flask import redirect
except Exception as e:
print ("Exception found:")
print (e)
try:
import main
except Exception as e:
print ("Exception found:")
print (e)
def test_database(self):
# global imports (dont't worry, red is normal)
try:
from typing import List, Dict
import network_constants as netconst
from database.MongoRepositoryBase import MongoRepositoryBase
from database.entities.transaction import Transaction
import pymongo
import json
import time
except Exception as e:
print ("Exception found:")
print (e)
try:
from database import repository
except Exception as e:
print ("Exception found:")
print (e)
def test_messaging(self):
try:
from security.token_manager import TokenManager
import network_constants
from database.entities.transaction import Transaction
from database.repository import Repository
from messaging.rest_fetcher import RestFetcher
import json
import hashlib
import logging
import requests
from typing import Dict
from typing import List
except Exception as e:
print ("Exception found:")
print (e)
try:
from messaging import MessageHandler
except Exception as e:
print ("Exception found:")
print (e)
try:
from messaging import rest_fetcher
except Exception as e:
print ("Exception found:")
print (e)
def test_routes(self):
#global imports
try:
from database.entities.transaction import Transaction
from database.repository import Repository
except Exception as e:
print ("Exception found:")
print (e)
try:
import json
from flask import Response, request
except Exception as e:
print ("Exception found:")
print (e)
try:
from routes import transactions
except Exception as e:
print ("Exception found:")
print (e)
if __name__ == '__main__':
unittest.main()
except Exception as e:
print ("Exception found:")
print (e)
\ No newline at end of file
src/transaction-hub-in/trace-retrieval-microservice/app/tests/test_MessageHandler.py
View file @ 271733ea
......@@ -10,6 +10,7 @@ class DummyMongoRepo:
def __init__(self):
self.added_transactions = []
self.duplicated_transactions = []
def insert_trace(self, trace):
self.last_trace = trace
......@@ -17,6 +18,22 @@ class DummyMongoRepo:
def add_transaction(self, transaction):
self.added_transactions.append(transaction)
def get_transaction_with_id(self, unique_id: str, use_case,table):
result = []
for trans in self.added_transactions:
transID = trans.id()
if transID == unique_id and trans.use_case == use_case and trans.table == table:
result.append(trans)
if len(result) > 0:
return result[0]
return None
def add_duplicated_transaction(self, transaction):
self.duplicated_transactions.append(transaction)
from messaging.DummyMessageManager import DummyMessageManager as DummyMessageSender
from messaging.dummy_rest_fetcher import DummyRestFetcher
......@@ -53,6 +70,48 @@ class Test_MessageHandler(unittest.TestCase):
}
}
return json.dumps(message_values)
def _get_valid_message2(self) -> str:
message_values = \
{ 'type': 'blockchain-transaction',
'content':
{
"ApplicationType": "string2",
"docType": "string",
"Metadata": {},
"ResourceIds": "string",
"ResourceMd5": "string",
"ResourceState": "string",
"Timestamp": "2019-08-27T14:00:48.587Z",
"TransactionFrom": "string",
"TransactionFromLatLng": "string",
"TransactionId": "string",
"TransactionTo": "string",
"TransactionToLatLng": "string",
"TransferredAsset": "string"
}
}
return json.dumps(message_values)
def _get_valid_message3(self) -> str:
message_values = \
{ 'type': 'blockchain-transaction',
'content':
{
"ApplicationType": "string",
"docType": "string2",
"Metadata": {},
"ResourceIds": "string",
"ResourceMd5": "string",
"ResourceState": "string",
"Timestamp": "2019-08-27T14:00:48.587Z",
"TransactionFrom": "string",
"TransactionFromLatLng": "string",
"TransactionId": "string",
"TransactionTo": "string",
"TransactionToLatLng": "string",
"TransferredAsset": "string"
}
}
return json.dumps(message_values)
def test_handleGeneric_emptyMessage_NotJsonError(self):
res = self.handler.handle_generic('')
......@@ -111,5 +170,52 @@ class Test_MessageHandler(unittest.TestCase):
self.assertEqual('semantic-linking', self.msg_sender.last_message['key'])
self.assertEqual('new-trace', json.loads(self.msg_sender.last_message['msg'])["type"])
def test_handleBlockchainTransaction_duplicateTrace_oneTransAddedToDuplicateRepo(self):
msg = self._get_valid_message()
msg2 = self._get_valid_message()
msg = eval(msg)
msg2 = eval(msg2)
self.handler.handle_blockchain_transaction(msg['content'])
self.handler.handle_blockchain_transaction(msg2['content'])
self.assertEqual(len(self.repo.added_transactions),len(self.repo.duplicated_transactions))
self.assertEqual(len(self.repo.added_transactions),1)
def test_handleBlockchainTransaction_duplicateTraceDifferentTable_bothTransactionsAddedAsUnique(self):
msg = self._get_valid_message()
msg2 = self._get_valid_message2()
msg = eval(msg)
msg2 = eval(msg2)
self.handler.handle_blockchain_transaction(msg['content'])
self.handler.handle_blockchain_transaction(msg2['content'])
self.assertEqual(len(self.repo.added_transactions),2)
def test_handleBlockchainTransaction_duplicateTraceDifferentUseCase_bothTransactionsAddedAsUnique(self):
msg = self._get_valid_message()
msg2 = self._get_valid_message3()
msg = eval(msg)
msg2 = eval(msg2)
self.handler.handle_blockchain_transaction(msg['content'])
self.handler.handle_blockchain_transaction(msg2['content'])
self.assertEqual(len(self.repo.added_transactions),2)
def test_handleBlockchainTransaction_multipleTransactions3SameIdDiffUseCaseTable_3AddedUnique2Duplicate(self):
#print("Entered Test: 3Unique 2Dupli")
msg = self._get_valid_message()
msg2 = self._get_valid_message2()
msg3 = self._get_valid_message3()
msg4 = self._get_valid_message3()
msg5 = self._get_valid_message3()
msg = eval(msg)
msg2 = eval(msg2)
msg3 = eval(msg3)
msg4 = eval(msg4)
msg5 = eval(msg5)
self.handler.handle_blockchain_transaction(msg['content'])
self.handler.handle_blockchain_transaction(msg2['content'])
self.handler.handle_blockchain_transaction(msg3['content'])
self.handler.handle_blockchain_transaction(msg4['content'])
self.handler.handle_blockchain_transaction(msg5['content'])
self.assertEqual(len(self.repo.added_transactions),3)
self.assertEqual(len(self.repo.duplicated_transactions),2)
if __name__ == '__main__':
unittest.main()
\ No newline at end of file
tools/check-use-case-data/check_mog.py 0 → 100644
View file @ 271733ea
import requests
requests.packages.urllib3.disable_warnings()
from icecream import ic
def httpget(url):
token = 'eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJ1c2VybmFtZSI6InJlZ3VsYXJAaXRlYy5hYXUuYXQiLCJjcmVhdGVkX2F0IjoiMjAyMS0wNC0wNyAwODo0MDo0NS4yODEwOTYiLCJ2YWxpZF91bnRpbCI6IjIwMjEtMDQtMDggMDg6NDA6NDUuMjgxMDk2In0.oIDOEYy8bmIR3AHDRU-T0upYU0Wcz7V4FYzO5tSaSzk'
res = requests.get(url,
verify=False,
headers = { "Authorization": f"Bearer {token}"})
return res
# list tables
res = httpget(url = 'https://articonf1.itec.aau.at:30420/api/use-cases/crowd-journalism/tables')
print("Tables: ", [entry['name'] for entry in res.json()])
# count pushed data
def count_data(json_res, table_identifier='table'):
tables = {}
for entry in json_res:
key = entry[table_identifier]
if key not in tables:
tables[key] = 0
tables[key] += 1
ic(tables)
res = httpget(url = 'https://articonf1.itec.aau.at:30001/api/use_cases/crowd-journalism/transactions')
count_data(res.json())
res_f = httpget(url = 'https://articonf1.itec.aau.at:30001/api/use_cases/crowd-journalism/transactions-failed')
count_data(res_f.json(), 'docType')
res_d = httpget(url = 'https://articonf1.itec.aau.at:30001/api/use_cases/crowd-journalism/transactions-duplicated')
count_data(res_d.json())
# failed tags: the "tag" is missing, but is called name
# failed purchases: duplicate keys generated from (userid, videoid, ownerid)
# failed classifications: impact is missing
# visualize content
import matplotlib.pyplot as plt
def visualize_video_coordinates():
geolocations = []
for entry in res.json():
if entry['table'] != 'video':
continue
loc_ = entry['properties']['geolocation'].split(',')
if loc_[0] == 'undefined' or loc_[1] == 'undefined':
continue
geolocations.append(loc_)
plt.scatter([float(coor[0]) for coor in geolocations], [float(coor[1]) for coor in geolocations])
plt.axis('off')
plt.show()
# visualize_video_coordinates()
def visualize_video_prices():
price = []
for entry in res.json():
if entry['table'] != 'video':
continue
price.append(entry['properties']['price'])
from collections import Counter
print(Counter(price))
plt.hist(price, bins=len(set(price)))
plt.show()
# visualize_video_prices()
def visualize_content_ratings():
impact = []
informative = []
trustiness = []
for entry in res.json():
if entry['table'] != 'classification':
continue
if entry['properties']['impact'] is not None:
impact.append(entry['properties']['impact'])
if entry['properties']['informative'] is not None:
informative.append(entry['properties']['informative'])
if entry['properties']['trustiness'] is not None:
trustiness.append(entry['properties']['trustiness'])
from collections import Counter
print(Counter(impact))
print(Counter(informative))
print(Counter(trustiness))
fig, (ax1, ax2, ax3) = plt.subplots(3)
ax1.hist(impact, bins=len(set(impact)))
ax1.set_title('impact')
ax2.hist(informative, bins=len(set(informative)))
ax2.set_title('informative')
ax3.hist(trustiness, bins=len(set(trustiness)))
ax3.set_title('trustiness')
plt.show()
# visualize_content_ratings()
# counting duplicate entries for 'purchase'
working_purchase_ids = [(entry['properties']['userid'], entry['properties']['videoid'], entry['properties']['ownerid']) for entry in res.json() if entry['table'] == 'purchase']
failed_purchase_ids = [(entry['userid'], entry['videoid'], entry['ownerid']) for entry in res_f.json() if entry['docType'] == 'purchase']
ic(len(working_purchase_ids))
ic(len(failed_purchase_ids))
cnt = 0
for failed_id in failed_purchase_ids:
if failed_id in working_purchase_ids:
cnt += 1
ic(cnt)
\ No newline at end of file
tools/check-use-case-data/reupload_failed_mog.py 0 → 100644
View file @ 271733ea
import requests
requests.packages.urllib3.disable_warnings()
from icecream import ic
token = 'eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJ1c2VybmFtZSI6InJlZ3VsYXJAaXRlYy5hYXUuYXQiLCJjcmVhdGVkX2F0IjoiMjAyMS0wMy0yNCAxMDoxMzo1MS4wMjkwNDkiLCJ2YWxpZF91bnRpbCI6IjIwMjEtMDMtMjUgMTA6MTM6NTEuMDI5MDQ5In0.V6kYV5Lmb_tUIsF-6AKNB8_lIifmJP_Dm8gHhGa5w_o'
def httpget(url):
res = requests.get(url,
verify=False,
headers = { "Authorization": f"Bearer {token}"})
return res
res_f = httpget(url = 'https://articonf1.itec.aau.at:30001/api/use_cases/crowd-journalism/transactions-failed')
failed_purchases = []
for entry in res_f.json():
if entry['docType'] == 'purchase':
failed_purchases.append(entry)
print(len(failed_purchases))
# upload again
def httppost_gateway(content_):
url = 'https://articonf1.itec.aau.at:30401/api/trace'
res = requests.post(url,
verify=False,
headers = { "Authorization": f"Bearer {token}"},
json=content_)
return res
for purchase in failed_purchases:
res = httppost_gateway(purchase)
print(res)
\ No newline at end of file
tools/reddit-upload/reddit_dataset.json 0 → 100644
View file @ 271733ea
This source diff could not be displayed because it is too large. You can view the blob instead.
tools/reddit-upload/upload_data.py 0 → 100644
View file @ 271733ea
''' This script adds all data from BitYoga's csv to our pipeline.'''
import csv
import requests
import sys
import os
import json
import urllib3
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
# modules_path = '../../../modules/'
# if os.path.exists(modules_path):
# sys.path.insert(1, modules_path)
# import network_constants as nc
# from security.token_manager import TokenManager
def send_transaction_to_rest_gateway(transaction: dict):
# token from Rest Gateway to authorize
JWT_TOKEN = 'eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJ1c2VybmFtZSI6InJlZ3VsYXJAaXRlYy5hYXUuYXQiLCJjcmVhdGVkX2F0IjoiMjAyMS0wMy0xNiAxMzoxNDoyMS42MDc1NjciLCJ2YWxpZF91bnRpbCI6IjIwMjEtMDMtMTcgMTM6MTQ6MjEuNjA3NTY3In0.ZGObriEDWYo1BgiYN3pQSosS7UuNrq10GSCSjmRHSAw'
res = requests.post(
url = 'https://articonf1.itec.aau.at:30401/api/trace',
json = transaction,
headers = {"Authorization": f"Bearer {JWT_TOKEN}"},
verify = False # ignore ssl error
)
if res.status_code >= 400:
raise Exception(f"Error while uploading: {str(res.content)}")
#print(res) Lots of spam
# file to read the data from
JSON_DATASET = r'reddit_dataset.json'
if __name__ == '__main__':
with open(JSON_DATASET, 'r') as json_file:
# reader = csv.reader(file)
json_data_list = json.load(json_file)
#titles = next(json_data)
summ = 0
for obj_dict in json_data_list:
transaction = {}
transaction['ApplicationType'] = 'reddit'
transaction['docType'] = 'reddit'
for key, value in obj_dict.items():
transaction[key] = value
send_transaction_to_rest_gateway(transaction)
summ+=1
if (summ % 1000 == 0 ):
print ("Uploaded " + str(summ) + " transactions.")
if summ >= 1:
break
print ("TOTAL Uploaded " + str(summ) + " transactions.")
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment