Commit fec1e504 authored by Alexander Lercher's avatar Alexander Lercher

Uploading data with updated user demand

parent e359a787
import sys
import os
modules_path = '../../../modules/'
if os.path.exists(modules_path):
sys.path.insert(1, modules_path)
import json
from db.entities import Cluster
from typing import List
from db.repository import Repository
repo = Repository()
def get_clusters(layer_file) -> List[Cluster]:
with open(layer_file, 'r') as file:
clusters = json.loads(file.read())
return [Cluster(cluster_dict=cluster, from_db=True) for cluster in clusters]
def store_generic_clusters(clusters: List[Cluster], layer):
print([c.label for c in clusters][0:10])
return
try:
with open(f'{layer}.json', 'w') as file:
cluster_dicts = [c.to_serializable_dict(for_db=False) for c in clusters]
file.write(json.dumps(cluster_dicts))
except:
print(f"failed writing {layer}")
try:
for cluster in clusters:
repo.add_cluster(cluster)
except:
print(f"failed uploading {layer}")
layers = ['Position_Layer.json', 'Solar_Production_Layer.json']
for layer in layers:
clusts: List[Cluster] = get_clusters(layer)
# print(len(clusts))
store_generic_clusters(clusts, layer)
\ No newline at end of file
...@@ -34,7 +34,7 @@ class Repository(MongoRepositoryBase): ...@@ -34,7 +34,7 @@ class Repository(MongoRepositoryBase):
def get_layers(self) -> List[Layer]: def get_layers(self) -> List[Layer]:
'''Retrieves all layers from the db, independent of use-case.''' '''Retrieves all layers from the db, independent of use-case.'''
entries = super().get_entries(self._layer_collection) entries = super().get_entries(self._layer_collection, projection={'_id': 0})
return [Layer(e) for e in entries] return [Layer(e) for e in entries]
def get_layers_for_use_case(self, use_case: str) -> Layer: def get_layers_for_use_case(self, use_case: str) -> Layer:
...@@ -81,6 +81,9 @@ class Repository(MongoRepositoryBase): ...@@ -81,6 +81,9 @@ class Repository(MongoRepositoryBase):
#endregion #endregion
#region Clusters #region Clusters
def add_cluster(self, cluster: Cluster):
super().insert_entry(self._clusters_collection, cluster.to_serializable_dict(for_db=True))
def add_clusters(self, clusters: List[Cluster]): def add_clusters(self, clusters: List[Cluster]):
cluster_dicts = [c.to_serializable_dict(for_db=True) for c in clusters] cluster_dicts = [c.to_serializable_dict(for_db=True) for c in clusters]
super().insert_many(self._clusters_collection, cluster_dicts) super().insert_many(self._clusters_collection, cluster_dicts)
......
...@@ -22,10 +22,10 @@ def upload_transaction(transaction): ...@@ -22,10 +22,10 @@ def upload_transaction(transaction):
uid = hashlib.sha256(f"{transaction['Customer']}{transaction['Timestamp']}".encode("utf-8")).hexdigest() uid = hashlib.sha256(f"{transaction['Customer']}{transaction['Timestamp']}".encode("utf-8")).hexdigest()
transaction['UniqueID'] = uid transaction['UniqueID'] = uid
t = { t = {
'use_case': 'smart-energy', 'use_case': transaction['ApplicationType'],
'table': 'smart-energy', 'table': transaction['docType'],
'id': uid, 'id': uid,
'properties': transaction 'properties': transaction,
} }
handler.handle_new_trace(t) handler.handle_new_trace(t)
...@@ -40,15 +40,16 @@ if __name__ == '__main__': ...@@ -40,15 +40,16 @@ if __name__ == '__main__':
for row in reader: for row in reader:
transaction = {} transaction = {}
transaction['ApplicationType'] = 'smart-energy' transaction['ApplicationType'] = 'smart-energy'
transaction['docType'] = 'smart-energy' transaction['docType'] = 'smart-energy-paper'
for idx in range(len(row)): for idx in range(len(row)):
transaction[titles[idx]] = row[idx] transaction[titles[idx]] = row[idx]
if int(transaction['Customer']) < 20: # also include the user demand, as Total_Demand_WMh is not per user
continue energy_cons = float(transaction['Energy_Consumption_kWh']) if transaction['Energy_Consumption_kWh'] is not None and transaction['Energy_Consumption_kWh'] != "" else 0
heating_cons = float(transaction['Heating_Consumption_kWh']) if transaction['Heating_Consumption_kWh'] is not None and transaction['Heating_Consumption_kWh'] != "" else 0
transaction['User_Demand_kWh'] = heating_cons + energy_cons
if transaction['Customer'] != old_c: if transaction['Customer'] != old_c:
# only upload until 200 for now
old_c = transaction['Customer'] old_c = transaction['Customer']
print(f"uploading for {old_c}") print(f"uploading for {old_c}")
......
...@@ -20,7 +20,8 @@ def add_table(use_case: str, table_name: str): ...@@ -20,7 +20,8 @@ def add_table(use_case: str, table_name: str):
"Price_AUD/MWh", "Price_AUD/MWh",
"Total_Demand_MWh", "Total_Demand_MWh",
"Latitude", "Latitude",
"Longitude" "Longitude",
"User_Demand_kWh",
] ]
columns = { c.replace("/", "_") : c for c in columns } columns = { c.replace("/", "_") : c for c in columns }
...@@ -119,7 +120,7 @@ def add_layers(use_case:str, table_name: str): ...@@ -119,7 +120,7 @@ def add_layers(use_case:str, table_name: str):
},{ },{
"use_case": use_case, "use_case": use_case,
"table": table_name, "table": table_name,
"name": "Position_Layer", "name": "Location_Layer",
"properties": [ "properties": [
"UniqueID", "UniqueID",
"Customer", "Customer",
...@@ -133,6 +134,21 @@ def add_layers(use_case:str, table_name: str): ...@@ -133,6 +134,21 @@ def add_layers(use_case:str, table_name: str):
"Longitude", "Longitude",
] ]
}, },
{
"use_case": use_case,
"table": table_name,
"name": "User_Demand_Layer",
"properties": [
"UniqueID",
"Customer",
"Postcode",
"Timestamp",
"User_Demand_kWh"
],
"cluster_properties": [
"User_Demand_kWh",
]
},
] ]
jwt = TokenManager.getInstance().getToken() jwt = TokenManager.getInstance().getToken()
...@@ -150,10 +166,8 @@ def add_layers(use_case:str, table_name: str): ...@@ -150,10 +166,8 @@ def add_layers(use_case:str, table_name: str):
print(url+": "+str(response.status_code)) print(url+": "+str(response.status_code))
def main(use_case: str): def main(use_case: str = "smart-energy", table_name: str = "smart-energy-paper"):
print("SMART-ENERGY") print("SMART-ENERGY")
table_name = "smart-energy"
add_table(use_case, table_name) add_table(use_case, table_name)
add_layers(use_case, table_name) add_layers(use_case, table_name)
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment