Commit e0cb3a2e authored by Alexander Lercher's avatar Alexander Lercher

Changed pipeline to generate local files

parent b0499784
# this file contains all the training data but is around 1.8 GB.
train.csv
# this folder contains all nodes generated from the upload script
nodes/
\ No newline at end of file
......@@ -94,3 +94,11 @@ if __name__ == '__main__':
if idx % 1000 == 0:
print(f"Progress: {str(float(idx) / 1710671)} %")
all_nodes = handler.get_all()
layers = {n['layer_name'] for n in all_nodes}
for l in layers:
layer_content = [e for e in all_nodes if e['layer_name'] == l]
with open(f'{l}.json', 'w') as f:
f.write(json.dumps(layer_content))
\ No newline at end of file
# this file contains all the training data but is large.
videos.csv
# this folder contains all nodes generated from the upload script
nodes/
\ No newline at end of file
......@@ -41,8 +41,8 @@ def upload_transaction(transaction):
del transaction['trending_timestamp']
t = {
'use_case': 'community-prediction-youtube-n',
'table': 'community-prediction-youtube-n',
'use_case': 'community-prediction-youtube',
'table': 'community-prediction-youtube',
'id': uid,
'properties': transaction,
}
......@@ -58,3 +58,7 @@ if __name__ == '__main__':
if idx % 1000 == 0:
print(f"Progress: {str(float(idx) / 375942)} %")
res = handler.get_all()
with open("res.json", 'w') as f:
f.write((json.dumps(res)))
\ No newline at end of file
......@@ -13,6 +13,7 @@ LOGGER = logging.getLogger(__name__)
class MessageHandler:
def __init__(self, repository):
self._repository = repository
self.allnodes = []
def handle_generic(self, body):
LOGGER.info(f"Received message: {body}")
......@@ -97,7 +98,14 @@ class MessageHandler:
nodes.append(node)
if len(nodes) > 0:
self._repository.add_layer_nodes(nodes)
self.store(nodes)
# self._repository.add_layer_nodes(nodes)
def store(self, nodes):
self.allnodes.extend(nodes)
def get_all(self):
return self.allnodes
def handle_new_traces_available(self):
# get all traces and call the Processor
......
......@@ -22,6 +22,7 @@ lazy-object-proxy==1.4.3
MarkupSafe==1.1.1
mccabe==0.6.1
openapi-spec-validator==0.2.8
pandas
pika==1.1.0
prance==0.19.0
pycparser==2.20
......
......@@ -7,14 +7,17 @@ import requests
modules_path = '../../../modules/'
if os.path.exists(modules_path):
sys.path.insert(1, modules_path)
sys.path.insert(1, '.')
import network_constants as nc
from security.token_manager import TokenManager
import tables.add_table as add_table
import _add_use_case_scripts.community_prediction.taxi.tables.add_table as add_table
# import tables.add_table as add_table
def add_use_case(use_case: str):
jwt = TokenManager.getInstance().getToken()
jwt = "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJ1c2VybmFtZSI6InJvb3RAaXRlYy5hYXUuYXQiLCJjcmVhdGVkX2F0IjoiMjAyMS0xMi0wOCAxNzo1OToxOC43OTMxMzUiLCJ2YWxpZF91bnRpbCI6IjIwMjEtMTItMDkgMTc6NTk6MTguNzkzMTM1In0.yQPH0KL3oaMyJAj_QcdEaSEWgSHSI-2qpck_aa3czHk"
url = f"https://articonf1.itec.aau.at:30420/api/use-cases"
response = requests.post(
url,
......@@ -24,7 +27,7 @@ def add_use_case(use_case: str):
json = {"name": use_case}
)
print(url+": "+str(response.status_code))
print(url+": "+str(response.content))
if __name__ == "__main__":
......
......@@ -6,7 +6,7 @@ import requests
def add_table(use_case: str, table_name: str):
''' Adds the use-case table with all the mappings as dict Internal -> External. '''
jwt = TokenManager.getInstance().getToken()
jwt = "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJ1c2VybmFtZSI6InJvb3RAaXRlYy5hYXUuYXQiLCJjcmVhdGVkX2F0IjoiMjAyMS0xMi0wOCAxNzo1OToxOC43OTMxMzUiLCJ2YWxpZF91bnRpbCI6IjIwMjEtMTItMDkgMTc6NTk6MTguNzkzMTM1In0.yQPH0KL3oaMyJAj_QcdEaSEWgSHSI-2qpck_aa3czHk"
mapping = { c : c for c in [
# mapping does not change any of the names for these properties
......@@ -44,7 +44,7 @@ def add_table(use_case: str, table_name: str):
def add_layers(use_case:str, table_name: str):
jwt = TokenManager.getInstance().getToken()
jwt = "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJ1c2VybmFtZSI6InJvb3RAaXRlYy5hYXUuYXQiLCJjcmVhdGVkX2F0IjoiMjAyMS0xMi0wOCAxNzo1OToxOC43OTMxMzUiLCJ2YWxpZF91bnRpbCI6IjIwMjEtMTItMDkgMTc6NTk6MTguNzkzMTM1In0.yQPH0KL3oaMyJAj_QcdEaSEWgSHSI-2qpck_aa3czHk"
layers = [
{
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment