Added Layer REST interface and database persistence

Moved REST routes

src/data-hub/community-detection-microservice/app/configs/swagger.yml

@@ -14,7 +14,7 @@ basePath: "/api"
 paths:
   /debug:
     post:
-      operationId: "rest.debug.echo"
+      operationId: "routes.debug.echo"
       tags:
         - "Echo"
       summary: "Echo function for debugging purposes"
@@ -31,7 +31,7 @@ paths:
           
   /locations:
     post:
-        operationId: "rest.location.post"
+        operationId: "routes.location.post"
         tags:
           - "Locations"
         summary: "Add new location data"
@@ -48,7 +48,7 @@ paths:
           400:
             description: "Invalid input"
     get:
-        operationId: "rest.location.get"
+        operationId: "routes.location.get"
         tags:
             - "Locations"
         summary: "Get location data"
@@ -61,7 +61,7 @@ paths:
   
   /location-collections:
     post:
-        operationId: "rest.location.post_many"
+        operationId: "routes.location.post_many"
         tags:
           - "Locations"
         summary: "Add new location data collection"
@@ -78,9 +78,39 @@ paths:
           400:
             description: "Invalid input"
 
+  /layers:
+    post:
+        operationId: "routes.layers.post"
+        tags:
+          - "Layers"
+        summary: "Add a new layer or overwrite an existing one"
+        parameters:
+          - in: body 
+            name: "Layer"
+            description: "The layer data to be added"
+            required: true
+            schema:
+                $ref: "#/definitions/Layer"
+        responses:
+            201:
+                description: "Successful operation"
+            400:
+                description: "Invalid input"
+    get:
+        operationId: "routes.layers.get"
+        tags:
+            - "Layers"
+        summary: "Get all layer data"
+        parameters: []
+        responses:
+            200:
+                description: "Successful operation"
+                schema:
+                    $ref: "#/definitions/Layer-pythonicCollection"
+
   /location-clusters:
     get:
-        operationId: "rest.cluster.get_locations"
+        operationId: "routes.cluster.get_locations"
         tags:
             - "Clusters"
         summary: "Get user communities clustered by location"
@@ -93,7 +123,7 @@ paths:
 
   # /clusters/cluster.png:
   #   get:
-  #       operationId: "rest.cluster.get_image"
+  #       operationId: "routes.cluster.get_image"
   #       tags:
   #           - "Clusters"
   #       summary: "Get user communities per date per hour as image"
@@ -106,7 +136,7 @@ paths:
   
   /time-clusters:
     get:
-        operationId: "rest.cluster.get_times"
+        operationId: "routes.cluster.get_times"
         tags:
             - "Clusters"
         summary: "Get user communities clustered by time per hour"
@@ -119,7 +149,7 @@ paths:
 
   # /agi/clusters/cluster.png:
   #   get:
-  #       operationId: "rest.agi_cluster.get_image"
+  #       operationId: "routes.agi_cluster.get_image"
   #       tags:
   #           - "Clusters"
   #       summary: "Get user communities per date per hour from agi data as image"
@@ -132,7 +162,7 @@ paths:
           
   /user-cluster-graphs:
     get:
-        operationId: "rest.user_cluster.get"
+        operationId: "routes.user_cluster.get"
         tags:
             - "User Graphs"
         summary: "Get user graphs per layer per cluster"
@@ -145,7 +175,7 @@ paths:
   
   /rfc/run:
     post:
-        operationId: "rest.functions.run_agi_clustering_and_graph_creation"
+        operationId: "routes.functions.run_agi_clustering_and_graph_creation"
         tags:
             - "Remote function calls"
         summary: "Insert locations from AGI, create clusters for starting time and location layers, create graphs for the location clusters"
@@ -235,4 +265,37 @@ definitions:
   UserClusterGraphCollection:
     type: array
     items:
-      $ref: "#/definitions/UserClusterGraph"
+      $ref: "#/definitions/UserClusterGraph"
\ No newline at end of file
+
+  Layer:
+    type: object
+    properties:
+        LayerName:
+            type: string
+        Nodes:
+            type: array
+            items: 
+                type: object
+        Properties:
+            type: array
+            items: 
+                type: string
+    
+  Layer-pythonic:
+    type: object
+    properties:
+        layer_name:
+            type: string
+        nodes:
+            type: array
+            items: 
+                type: object
+        properties:
+            type: array
+            items: 
+                type: string
+
+  Layer-pythonicCollection:
+    type: array
+    items:
+        $ref: "#/definitions/Layer-pythonic"
\ No newline at end of file

src/data-hub/community-detection-microservice/app/db/agi/agi_repository.py

@@ -32,18 +32,18 @@ class AgiRepository:
     def getLocationsBasedOnNewDataSchema(self):
         '''Creates the new data generic schema to be used beginning on 24.03.2020'''
         data = {
-            'LayerName': 'Destination',
-            'Nodes': self.getLocations(),
-            'Properties': ['latitude', 'longitude']
+            'layer_name': 'Destination',
+            'nodes': self.getLocations(),
+            'properties': ['latitude', 'longitude']
         }
         return data
         
     def getTimesBasedOnNewDataSchema(self):
         '''Creates the new data generic schema to be used beginning on 24.03.2020'''
         data = {
-            'LayerName': 'Starting_Time',
-            'Nodes': self.getLocations(),
-            'Properties': ['timestamp']
+            'layer_name': 'Starting_Time',
+            'nodes': self.getLocations(),
+            'properties': ['timestamp']
         }
         return data
 

src/data-hub/community-detection-microservice/app/db/entities/__init__.py

 from db.entities.location import Location
 from db.entities.popular_location import PopularLocation
 from db.entities.cluster import Cluster, LocationCluster, TimeCluster
-from db.entities.user_cluster_graph import UserClusterGraph
+from db.entities.user_cluster_graph import UserClusterGraph
\ No newline at end of file
+from db.entities.layer import Layer
\ No newline at end of file

src/data-hub/community-detection-microservice/app/db/entities/layer.py

+import json
+from datetime import datetime
+from typing import Dict
+
+
+class Layer:
+    '''
+    This class represents a single layer of the Multilayer Graph.
+
+    :param layer_info: Information as dictionary to restore the layer object.
+    '''
+
+    def __init__(self, layer_info: Dict = None, from_db=False):
+        if layer_info is not None:
+            self.from_serializable_dict(layer_info, from_db)
+
+    def to_serializable_dict(self, for_db=False) -> Dict:
+        return {
+            "layer_name": self.layer_name,
+            "properties": self.properties,
+            "nodes": json.dumps(self.nodes) if for_db else self.nodes
+        }
+
+    def from_serializable_dict(self, layer_info: Dict, from_db=False):
+        self.layer_name = layer_info['layer_name']
+        self.properties = layer_info['properties']
+        self.nodes = json.loads(layer_info["nodes"]) \
+            if from_db else layer_info["nodes"]
+
+    def __repr__(self):
+        return json.dumps(self.to_serializable_dict())
+
+    def __str__(self):
+        return f"Layer({self.__repr__()})"
+
+
+layer_d = {
+"layer_name": "Destination",
+"nodes": [ {
+            "TravelID": 1,
+            "UserID": "Micah",
+            "Latitude_Destination": -5.95081,
+            "Longitude_Destination": 37.415281,
+            "Finished_time": 1579143634812589,
+            "TravelPrice": 19
+        }],
+"properties": ['Latitude_StartingPoint', 'Longitude_StartingPoint']
+}
+
+layer = Layer(layer_d)
+print(layer.to_serializable_dict(for_db=True))
\ No newline at end of file

src/data-hub/community-detection-microservice/app/db/repository.py

@@ -5,12 +5,12 @@ import json
 
 from db.agi.agi_repository import AgiRepository
 
-from db.entities import Location, TimeCluster, PopularLocation, LocationCluster, UserClusterGraph
+from db.entities import *
 from typing import List
 
 
 class Repository(MongoRepositoryBase):
-    '''This repository stores and loads locations and clusters with MongoDb.'''
+    '''This is a repository for MongoDb.'''
 
     def __init__(self):
         super().__init__(netconst.COMMUNITY_DETECTION_DB_HOSTNAME,
@@ -21,6 +21,7 @@ class Repository(MongoRepositoryBase):
         self._location_cluster_collection = 'location_cluster'
         self._time_cluster_collection = 'time_cluster'
         self._user_cluster_graph_collection = 'user_cluster_graph'
+        self._layer_collection = 'layer'
 
         self.agi_repo = AgiRepository()
 
@@ -58,3 +59,10 @@ class Repository(MongoRepositoryBase):
     def get_user_cluster_graphs(self) -> List[UserClusterGraph]:
         user_graphs = super().get_entries(self._user_cluster_graph_collection)
         return [UserClusterGraph(dict_=u, from_db=True) for u in user_graphs]
+
+    def add_layer(self, layer: Layer):
+        super().insert_entry(self._layer_collection, layer.to_serializable_dict())
+
+    def get_layers(self) -> List[Layer]:
+        entries = super().get_entries(self._layer_collection)
+        return [Layer(e) for e in entries]
\ No newline at end of file

src/data-hub/community-detection-microservice/app/processing/clustering/clusterer.py

@@ -153,22 +153,22 @@ if __name__ == '__main__':
     agi_repo = AgiRepository()
 
     if True:
-        res_old = clusterer.cluster_locations(agi_repo.getLocationsBasedOnNewDataSchema()['Nodes'])
+        res_old = clusterer.cluster_locations(agi_repo.getLocationsBasedOnNewDataSchema()['nodes'])
         # print(res_old[11])
         # [{'id': 'adad64cb-bd71-4b2b-9a70-e08eb8b19901-1570900602', 'latitude': -20.2695062, 'longitude': 57.6297389, 'timestamp': 1570900602, 'user': 'b57ad1fb396cfc18b8867fb2e08be723c2cdc2a6', 'cluster_label': 11}, {'id': '127af17b-e823-4d30-8227-00f5421bd48b-1549291309', 'latitude': -20.5362627, 'longitude': 47.2459749, 'timestamp': 1549291309, 'user': 'ca34bd51c4dc65cbc021cb27bcaa014ca082b8c4', 'cluster_label': 11}]
 
     data = agi_repo.getLocationsBasedOnNewDataSchema()
-    res = clusterer.cluster_dataset(data['Nodes'], data['Properties'])
+    res = clusterer.cluster_dataset(data['nodes'], data['properties'])
     # if res is not None:
     #     print(res[11])
 
     assert (res_old == res)
 
     # time 
-    res_old = clusterer.cluster_times(agi_repo.getTimesBasedOnNewDataSchema()['Nodes'])
+    res_old = clusterer.cluster_times(agi_repo.getTimesBasedOnNewDataSchema()['nodes'])
 
     data = agi_repo.getTimesBasedOnNewDataSchema()
-    res = clusterer.cluster_dataset(data['Nodes'], data['Properties'])
+    res = clusterer.cluster_dataset(data['nodes'], data['properties'])
     print(res_old[20])
     print(res[20])
     

src/data-hub/community-detection-microservice/app/requirements.txt

@@ -5,9 +5,11 @@ certifi==2019.11.28
 chardet==3.0.4
 Click==7.0
 clickclick==1.2.2
+colorama==0.4.3
 connexion==2.6.0
 cycler==0.10.0
 decorator==4.4.1
+Deprecated==1.2.7
 Flask==1.1.1
 idna==2.8
 importlib-metadata==1.5.0

src/data-hub/community-detection-microservice/app/routes/layers.py

+from flask import request, Response
+from db.repository import Repository
+from db.entities import Layer
+
+repo = Repository()
+
+def post():
+    body = request.json
+    _insert_layer(body)
+    return Response(status=201)
+
+def get():
+    return [l.to_serializable_dict() for l in repo.get_layers()]
+
+def _insert_layer(layer_data: dict):
+    # convert object keys from ext source
+    layer_data['layer_name'] = layer_data.pop('LayerName')
+    layer_data['nodes'] = layer_data.pop('Nodes')
+    layer_data['properties'] = layer_data.pop('Properties')
+
+    repo.add_layer(Layer(layer_data))