Uploading data with updated user demand

src/data-hub/role-stage-discovery-microservice/app/convert_layers.py

+import sys
+import os
+modules_path = '../../../modules/'
+if os.path.exists(modules_path):
+    sys.path.insert(1, modules_path)
+
+import json
+from db.entities import Cluster
+from typing import List
+from db.repository import Repository
+
+
+repo = Repository()
+
+def get_clusters(layer_file) -> List[Cluster]:
+    with open(layer_file, 'r') as file:
+        clusters = json.loads(file.read())
+
+        return [Cluster(cluster_dict=cluster, from_db=True) for cluster in clusters]
+
+
+def store_generic_clusters(clusters: List[Cluster], layer):
+
+    print([c.label for c in clusters][0:10])
+    return
+
+    try:
+        with open(f'{layer}.json', 'w') as file:
+            cluster_dicts = [c.to_serializable_dict(for_db=False) for c in clusters]
+            file.write(json.dumps(cluster_dicts))
+    except:
+        print(f"failed writing {layer}")
+
+    try:
+        for cluster in clusters:
+            repo.add_cluster(cluster)
+    except:
+        print(f"failed uploading {layer}")
+
+
+layers = ['Position_Layer.json', 'Solar_Production_Layer.json']
+for layer in layers:
+    clusts: List[Cluster] = get_clusters(layer)
+    # print(len(clusts))
+
+    store_generic_clusters(clusts, layer)
\ No newline at end of file

src/data-hub/role-stage-discovery-microservice/app/db/repository.py

@@ -34,7 +34,7 @@ class Repository(MongoRepositoryBase):
 
     def get_layers(self) -> List[Layer]:
         '''Retrieves all layers from the db, independent of use-case.'''
-        entries = super().get_entries(self._layer_collection)
+        entries = super().get_entries(self._layer_collection, projection={'_id': 0})
         return [Layer(e) for e in entries]
 
     def get_layers_for_use_case(self, use_case: str) -> Layer:
@@ -81,6 +81,9 @@ class Repository(MongoRepositoryBase):
 #endregion
 
 #region Clusters
+    def add_cluster(self, cluster: Cluster):
+        super().insert_entry(self._clusters_collection, cluster.to_serializable_dict(for_db=True))
+        
     def add_clusters(self, clusters: List[Cluster]):
         cluster_dicts = [c.to_serializable_dict(for_db=True) for c in clusters]
         super().insert_many(self._clusters_collection, cluster_dicts)

src/data-hub/semantic-linking-microservice/app/dummy_upload.py

@@ -22,10 +22,10 @@ def upload_transaction(transaction):
     uid = hashlib.sha256(f"{transaction['Customer']}{transaction['Timestamp']}".encode("utf-8")).hexdigest()
     transaction['UniqueID'] = uid
     t = {        
-        'use_case': 'smart-energy',
-        'table': 'smart-energy',
+        'use_case': transaction['ApplicationType'],
+        'table': transaction['docType'],
         'id': uid,
-        'properties': transaction
+        'properties': transaction,
     }
     handler.handle_new_trace(t)
 
@@ -40,15 +40,16 @@ if __name__ == '__main__':
         for row in reader:
             transaction = {}
             transaction['ApplicationType'] = 'smart-energy'
-            transaction['docType'] = 'smart-energy'
+            transaction['docType'] = 'smart-energy-paper'
             for idx in range(len(row)):
                 transaction[titles[idx]] = row[idx]
 
-            if int(transaction['Customer']) < 20:
-                continue
+            # also include the user demand, as Total_Demand_WMh is not per user
+            energy_cons = float(transaction['Energy_Consumption_kWh']) if transaction['Energy_Consumption_kWh'] is not None and transaction['Energy_Consumption_kWh'] != "" else 0
+            heating_cons = float(transaction['Heating_Consumption_kWh']) if transaction['Heating_Consumption_kWh'] is not None and transaction['Heating_Consumption_kWh'] != "" else 0
+            transaction['User_Demand_kWh'] = heating_cons + energy_cons
 
             if transaction['Customer'] != old_c:      
-                # only upload until 200 for now
                 old_c = transaction['Customer']
                 print(f"uploading for {old_c}")
 

src/participation-hub/business-logic-microservice/app/_add_use_case_scripts/smart-energy/tables/add_smart_energy.py

@@ -20,7 +20,8 @@ def add_table(use_case: str, table_name: str):
         "Price_AUD/MWh", 
         "Total_Demand_MWh", 
         "Latitude", 
-        "Longitude"
+        "Longitude",
+        "User_Demand_kWh",
     ]
     
     columns = { c.replace("/", "_") : c for c in columns }
@@ -119,7 +120,7 @@ def add_layers(use_case:str, table_name: str):
         },{
             "use_case": use_case,
             "table": table_name,
-            "name": "Position_Layer",
+            "name": "Location_Layer",
             "properties": [
                 "UniqueID",
                 "Customer",
@@ -133,6 +134,21 @@ def add_layers(use_case:str, table_name: str):
                 "Longitude",
             ]
         },
+        {
+            "use_case": use_case,
+            "table": table_name,
+            "name": "User_Demand_Layer",
+            "properties": [
+                "UniqueID",
+                "Customer",
+                "Postcode",
+                "Timestamp",
+                "User_Demand_kWh"
+            ],
+            "cluster_properties": [
+                "User_Demand_kWh",
+            ]
+        },
     ]
 
     jwt = TokenManager.getInstance().getToken()
@@ -150,10 +166,8 @@ def add_layers(use_case:str, table_name: str):
 
         print(url+": "+str(response.status_code))
 
-def main(use_case: str):
+def main(use_case: str = "smart-energy", table_name: str = "smart-energy-paper"):
     print("SMART-ENERGY")
 
-    table_name = "smart-energy"
-
     add_table(use_case, table_name)
     add_layers(use_case, table_name)
\ No newline at end of file