Added smart-energy direct upload

src/data-hub/semantic-linking-microservice/app/db/repository.py

@@ -21,8 +21,6 @@ class Repository(MongoRepositoryBase):
                          'semanticLinkingDb')
         self._layer_collection = 'layers'
         self._layer_nodes_collection = 'layer_nodes'
-        self._clusters_collection = 'clusters'
-        self._time_slice_collection = 'time_slices'
 
 
     # region Layers

src/data-hub/semantic-linking-microservice/app/dummy_upload.py

+import csv
+
+import sys
+import os
+modules_path = '../../../modules/'
+if os.path.exists(modules_path):
+    sys.path.insert(1, modules_path)
+
+from messaging.MessageHandler import MessageHandler
+
+# file to read the data from
+CSV_FILE = r'Energy_Dataset.csv'
+handler = MessageHandler()
+
+
+def upload_transaction(transaction):
+    '''{"type": "new-trace", 
+    "content": {"use_case": "smart-energy", "table": "smart-energy", "id": "dd2c5146c919b046d77a32a5cf553d5133163562f7b7e1298c878c575d516025", 
+    "properties": {"Customer": "297", "Energy_Consumption_kWh": "0.177", "Heating_Consumption_kWh": "0.0", "Latitude": "-33.362679", "Longitude": "151.447302", "Postcode": "2261", "Price_AUD_MWh": "58.05", "Solar_Production_kWh": "0.0", "Timestamp": "2013-06-30 00:00:00", "Total_Demand_MWh": "8154.14", "UniqueID": "dd2c5146c919b046d77a32a5cf553d5133163562f7b7e1298c878c575d516025"}
+    }}'''
+    uid = hash(str(transaction['Customer']) + str(transaction['Timestamp']))
+    transaction['UniqueID'] = uid
+    t = {        
+        'use_case': 'smart-energy',
+        'table': 'smart-energy',
+        'id': uid,
+        'properties': transaction
+    }
+    handler.handle_new_trace(t)
+
+
+if __name__ == '__main__':    
+    with open(CSV_FILE, 'r') as file:
+        reader = csv.reader(file)
+
+        titles = next(reader)
+
+        old_c = None
+        for row in reader:
+            transaction = {}
+            transaction['ApplicationType'] = 'smart-energy'
+            transaction['docType'] = 'smart-energy'
+            for idx in range(len(row)):
+                transaction[titles[idx]] = row[idx]
+
+            if transaction['Customer'] != old_c:      
+                # only upload until 200 for now
+                old_c = transaction['Customer']
+                print(f"uploading for {old_c}")
+
+            upload_transaction(transaction)
+

src/data-hub/semantic-linking-microservice/app/messaging/MessageHandler.py

@@ -64,7 +64,7 @@ class MessageHandler:
         return layers
 
     def handle_new_trace(self, content: Dict):
-        if "use_case" not in content.keys() or "id" not in content.keys() or "properties" not in content.keys() or "table" not in content.keys():
+        if "use_case" not in content or "id" not in content or "properties" not in content or "table" not in content:
             LOGGER.error(f"Missing fields in trace, required fields: (use_case, id, properties, table), given fields: ({content.keys()})")
             return