Commit e0cb3a2e authored by Alexander Lercher's avatar Alexander Lercher

Changed pipeline to generate local files

parent b0499784
# this file contains all the training data but is around 1.8 GB. # this file contains all the training data but is around 1.8 GB.
train.csv train.csv
\ No newline at end of file
# this folder contains all nodes generated from the upload script
nodes/
\ No newline at end of file
...@@ -93,4 +93,12 @@ if __name__ == '__main__': ...@@ -93,4 +93,12 @@ if __name__ == '__main__':
upload_transaction(transaction) upload_transaction(transaction)
if idx % 1000 == 0: if idx % 1000 == 0:
print(f"Progress: {str(float(idx) / 1710671)} %") print(f"Progress: {str(float(idx) / 1710671)} %")
\ No newline at end of file
all_nodes = handler.get_all()
layers = {n['layer_name'] for n in all_nodes}
for l in layers:
layer_content = [e for e in all_nodes if e['layer_name'] == l]
with open(f'{l}.json', 'w') as f:
f.write(json.dumps(layer_content))
\ No newline at end of file
# this file contains all the training data but is large. # this file contains all the training data but is large.
videos.csv videos.csv
\ No newline at end of file
# this folder contains all nodes generated from the upload script
nodes/
\ No newline at end of file
...@@ -41,8 +41,8 @@ def upload_transaction(transaction): ...@@ -41,8 +41,8 @@ def upload_transaction(transaction):
del transaction['trending_timestamp'] del transaction['trending_timestamp']
t = { t = {
'use_case': 'community-prediction-youtube-n', 'use_case': 'community-prediction-youtube',
'table': 'community-prediction-youtube-n', 'table': 'community-prediction-youtube',
'id': uid, 'id': uid,
'properties': transaction, 'properties': transaction,
} }
...@@ -57,4 +57,8 @@ if __name__ == '__main__': ...@@ -57,4 +57,8 @@ if __name__ == '__main__':
upload_transaction(transaction) upload_transaction(transaction)
if idx % 1000 == 0: if idx % 1000 == 0:
print(f"Progress: {str(float(idx) / 375942)} %") print(f"Progress: {str(float(idx) / 375942)} %")
\ No newline at end of file
res = handler.get_all()
with open("res.json", 'w') as f:
f.write((json.dumps(res)))
\ No newline at end of file
...@@ -13,6 +13,7 @@ LOGGER = logging.getLogger(__name__) ...@@ -13,6 +13,7 @@ LOGGER = logging.getLogger(__name__)
class MessageHandler: class MessageHandler:
def __init__(self, repository): def __init__(self, repository):
self._repository = repository self._repository = repository
self.allnodes = []
def handle_generic(self, body): def handle_generic(self, body):
LOGGER.info(f"Received message: {body}") LOGGER.info(f"Received message: {body}")
...@@ -97,7 +98,14 @@ class MessageHandler: ...@@ -97,7 +98,14 @@ class MessageHandler:
nodes.append(node) nodes.append(node)
if len(nodes) > 0: if len(nodes) > 0:
self._repository.add_layer_nodes(nodes) self.store(nodes)
# self._repository.add_layer_nodes(nodes)
def store(self, nodes):
self.allnodes.extend(nodes)
def get_all(self):
return self.allnodes
def handle_new_traces_available(self): def handle_new_traces_available(self):
# get all traces and call the Processor # get all traces and call the Processor
......
...@@ -22,6 +22,7 @@ lazy-object-proxy==1.4.3 ...@@ -22,6 +22,7 @@ lazy-object-proxy==1.4.3
MarkupSafe==1.1.1 MarkupSafe==1.1.1
mccabe==0.6.1 mccabe==0.6.1
openapi-spec-validator==0.2.8 openapi-spec-validator==0.2.8
pandas
pika==1.1.0 pika==1.1.0
prance==0.19.0 prance==0.19.0
pycparser==2.20 pycparser==2.20
......
...@@ -7,14 +7,17 @@ import requests ...@@ -7,14 +7,17 @@ import requests
modules_path = '../../../modules/' modules_path = '../../../modules/'
if os.path.exists(modules_path): if os.path.exists(modules_path):
sys.path.insert(1, modules_path) sys.path.insert(1, modules_path)
sys.path.insert(1, '.')
import network_constants as nc import network_constants as nc
from security.token_manager import TokenManager from security.token_manager import TokenManager
import tables.add_table as add_table import _add_use_case_scripts.community_prediction.taxi.tables.add_table as add_table
# import tables.add_table as add_table
def add_use_case(use_case: str): def add_use_case(use_case: str):
jwt = TokenManager.getInstance().getToken() jwt = "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJ1c2VybmFtZSI6InJvb3RAaXRlYy5hYXUuYXQiLCJjcmVhdGVkX2F0IjoiMjAyMS0xMi0wOCAxNzo1OToxOC43OTMxMzUiLCJ2YWxpZF91bnRpbCI6IjIwMjEtMTItMDkgMTc6NTk6MTguNzkzMTM1In0.yQPH0KL3oaMyJAj_QcdEaSEWgSHSI-2qpck_aa3czHk"
url = f"https://articonf1.itec.aau.at:30420/api/use-cases" url = f"https://articonf1.itec.aau.at:30420/api/use-cases"
response = requests.post( response = requests.post(
url, url,
...@@ -24,7 +27,7 @@ def add_use_case(use_case: str): ...@@ -24,7 +27,7 @@ def add_use_case(use_case: str):
json = {"name": use_case} json = {"name": use_case}
) )
print(url+": "+str(response.status_code)) print(url+": "+str(response.content))
if __name__ == "__main__": if __name__ == "__main__":
......
...@@ -6,7 +6,7 @@ import requests ...@@ -6,7 +6,7 @@ import requests
def add_table(use_case: str, table_name: str): def add_table(use_case: str, table_name: str):
''' Adds the use-case table with all the mappings as dict Internal -> External. ''' ''' Adds the use-case table with all the mappings as dict Internal -> External. '''
jwt = TokenManager.getInstance().getToken() jwt = "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJ1c2VybmFtZSI6InJvb3RAaXRlYy5hYXUuYXQiLCJjcmVhdGVkX2F0IjoiMjAyMS0xMi0wOCAxNzo1OToxOC43OTMxMzUiLCJ2YWxpZF91bnRpbCI6IjIwMjEtMTItMDkgMTc6NTk6MTguNzkzMTM1In0.yQPH0KL3oaMyJAj_QcdEaSEWgSHSI-2qpck_aa3czHk"
mapping = { c : c for c in [ mapping = { c : c for c in [
# mapping does not change any of the names for these properties # mapping does not change any of the names for these properties
...@@ -44,7 +44,7 @@ def add_table(use_case: str, table_name: str): ...@@ -44,7 +44,7 @@ def add_table(use_case: str, table_name: str):
def add_layers(use_case:str, table_name: str): def add_layers(use_case:str, table_name: str):
jwt = TokenManager.getInstance().getToken() jwt = "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJ1c2VybmFtZSI6InJvb3RAaXRlYy5hYXUuYXQiLCJjcmVhdGVkX2F0IjoiMjAyMS0xMi0wOCAxNzo1OToxOC43OTMxMzUiLCJ2YWxpZF91bnRpbCI6IjIwMjEtMTItMDkgMTc6NTk6MTguNzkzMTM1In0.yQPH0KL3oaMyJAj_QcdEaSEWgSHSI-2qpck_aa3czHk"
layers = [ layers = [
{ {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment