Commit fec1e504 authored by Alexander Lercher's avatar Alexander Lercher

Uploading data with updated user demand

parent e359a787
import sys
import os
modules_path = '../../../modules/'
if os.path.exists(modules_path):
sys.path.insert(1, modules_path)
import json
from db.entities import Cluster
from typing import List
from db.repository import Repository
repo = Repository()
def get_clusters(layer_file) -> List[Cluster]:
with open(layer_file, 'r') as file:
clusters = json.loads(file.read())
return [Cluster(cluster_dict=cluster, from_db=True) for cluster in clusters]
def store_generic_clusters(clusters: List[Cluster], layer):
print([c.label for c in clusters][0:10])
return
try:
with open(f'{layer}.json', 'w') as file:
cluster_dicts = [c.to_serializable_dict(for_db=False) for c in clusters]
file.write(json.dumps(cluster_dicts))
except:
print(f"failed writing {layer}")
try:
for cluster in clusters:
repo.add_cluster(cluster)
except:
print(f"failed uploading {layer}")
layers = ['Position_Layer.json', 'Solar_Production_Layer.json']
for layer in layers:
clusts: List[Cluster] = get_clusters(layer)
# print(len(clusts))
store_generic_clusters(clusts, layer)
\ No newline at end of file
......@@ -34,7 +34,7 @@ class Repository(MongoRepositoryBase):
def get_layers(self) -> List[Layer]:
'''Retrieves all layers from the db, independent of use-case.'''
entries = super().get_entries(self._layer_collection)
entries = super().get_entries(self._layer_collection, projection={'_id': 0})
return [Layer(e) for e in entries]
def get_layers_for_use_case(self, use_case: str) -> Layer:
......@@ -81,6 +81,9 @@ class Repository(MongoRepositoryBase):
#endregion
#region Clusters
def add_cluster(self, cluster: Cluster):
super().insert_entry(self._clusters_collection, cluster.to_serializable_dict(for_db=True))
def add_clusters(self, clusters: List[Cluster]):
cluster_dicts = [c.to_serializable_dict(for_db=True) for c in clusters]
super().insert_many(self._clusters_collection, cluster_dicts)
......
......@@ -22,10 +22,10 @@ def upload_transaction(transaction):
uid = hashlib.sha256(f"{transaction['Customer']}{transaction['Timestamp']}".encode("utf-8")).hexdigest()
transaction['UniqueID'] = uid
t = {
'use_case': 'smart-energy',
'table': 'smart-energy',
'use_case': transaction['ApplicationType'],
'table': transaction['docType'],
'id': uid,
'properties': transaction
'properties': transaction,
}
handler.handle_new_trace(t)
......@@ -40,15 +40,16 @@ if __name__ == '__main__':
for row in reader:
transaction = {}
transaction['ApplicationType'] = 'smart-energy'
transaction['docType'] = 'smart-energy'
transaction['docType'] = 'smart-energy-paper'
for idx in range(len(row)):
transaction[titles[idx]] = row[idx]
if int(transaction['Customer']) < 20:
continue
# also include the user demand, as Total_Demand_WMh is not per user
energy_cons = float(transaction['Energy_Consumption_kWh']) if transaction['Energy_Consumption_kWh'] is not None and transaction['Energy_Consumption_kWh'] != "" else 0
heating_cons = float(transaction['Heating_Consumption_kWh']) if transaction['Heating_Consumption_kWh'] is not None and transaction['Heating_Consumption_kWh'] != "" else 0
transaction['User_Demand_kWh'] = heating_cons + energy_cons
if transaction['Customer'] != old_c:
# only upload until 200 for now
old_c = transaction['Customer']
print(f"uploading for {old_c}")
......
......@@ -20,7 +20,8 @@ def add_table(use_case: str, table_name: str):
"Price_AUD/MWh",
"Total_Demand_MWh",
"Latitude",
"Longitude"
"Longitude",
"User_Demand_kWh",
]
columns = { c.replace("/", "_") : c for c in columns }
......@@ -119,7 +120,7 @@ def add_layers(use_case:str, table_name: str):
},{
"use_case": use_case,
"table": table_name,
"name": "Position_Layer",
"name": "Location_Layer",
"properties": [
"UniqueID",
"Customer",
......@@ -133,6 +134,21 @@ def add_layers(use_case:str, table_name: str):
"Longitude",
]
},
{
"use_case": use_case,
"table": table_name,
"name": "User_Demand_Layer",
"properties": [
"UniqueID",
"Customer",
"Postcode",
"Timestamp",
"User_Demand_kWh"
],
"cluster_properties": [
"User_Demand_kWh",
]
},
]
jwt = TokenManager.getInstance().getToken()
......@@ -150,10 +166,8 @@ def add_layers(use_case:str, table_name: str):
print(url+": "+str(response.status_code))
def main(use_case: str):
def main(use_case: str = "smart-energy", table_name: str = "smart-energy-paper"):
print("SMART-ENERGY")
table_name = "smart-energy"
add_table(use_case, table_name)
add_layers(use_case, table_name)
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment