Merge branch 'feature/federated-learning' into develop

src/modules/network_constants.py

@@ -84,4 +84,31 @@ else:
     BUSINESS_LOGIC_REST_PORT = 30420
     BUSINESS_LOGIC_DB_PORT = 30421
 
+## Federated Learning
+if server:
+    FEDERATED_LEARNING_HOSTNAME = 'federated-learning'
+    #FEDERATED_LEARNING_DB_HOSTNAME = f'{EDERATED_LEARNING_HOSTNAME}-db'
+    FEDERATED_LEARNING_REST_PORT = 80
+    #FEDERATED_LEARNING_DB_PORT = 27017
+else:
+    FEDERATED_LEARNING_HOSTNAME = 'articonf1.itec.aau.at'
+    #FEDERATED_LEARNING_DB_HOSTNAME = 'articonf1.itec.aau.at'
+    FEDERATED_LEARNING_REST_PORT = 30422
+    #FEDERATED_LEARNING_DB_PORT = 30423
+
 #endregion Participation Hub
+
+#region Federated Training
+## Federated Training
+if server:
+    FEDERATED_TRAINING_HOSTNAME = 'gpu3.itec.aau.at'
+    #FEDERATED_TRAINING_DB_HOSTNAME = f'{FEDERATED_TRAINING_HOSTNAME}-db'
+    FEDERATED_TRAINING_REST_PORT = 30424
+    #FEDERATED_TRAINING_DB_PORT = 27017
+else:
+    FEDERATED_TRAINING_HOSTNAME = 'gpu3.itec.aau.at'
+    #FEDERATED_TRAINING_DB_HOSTNAME = 'articonf1.itec.aau.at'
+    FEDERATED_TRAINING_REST_PORT = 30424
+    #FEDERATED_TRAINING_DB_PORT = 30425
+
+#endregion Federated Training
\ No newline at end of file

src/participation-hub/federated-learning-microservice/Dockerfile

+FROM python:3
+LABEL maintainer="Alexander Lercher"
+
+#ENV http_proxy http://proxy.uni-klu.ac.at:3128/
+#ENV https_proxy http://proxy.uni-klu.ac.at:3128/
+
+RUN apt-get update
+
+EXPOSE 5000
+WORKDIR /app
+
+COPY src/participation-hub/federated-learning-microservice/app/requirements.txt /app/
+RUN pip install -r requirements.txt
+
+COPY src/modules/ /app/
+COPY src/participation-hub/federated-learning-microservice/app/ /app/
+RUN chmod a+x main.py
+
+CMD ["python", "./main.py"]
\ No newline at end of file

src/participation-hub/federated-learning-microservice/README.md

+# Federated Learning Microservice
+The Federated Learning microservice serves as an interface for the Federated TRAINING microservice which handles all the federated processing in the back-end.
+
+## Technologies
+- Python 3.x
+- Docker
+- Kubernetes
\ No newline at end of file

src/participation-hub/federated-learning-microservice/app/configs/routes.yml

+swagger: '2.0'
+paths:
+  #####
+  # Owners
+  #####
+  /Owners/use_case/{use_case}/last_train:
+    get:
+      security:
+        - JwtRegular: []
+      operationId: "routes.owners.last"
+      tags:
+        - "Owners"
+      summary: "Get last train session data"
+      description: "Get last train session data"
+      parameters:
+        - name: "use_case"
+          in: "path"
+          description: "Name of the Use-Case to get the last training session from (must already exist)"
+          required: true
+          type: "string"
+      responses:
+        '200':
+          description: "Successful Request"
+        '404':
+          description: "Use case train session data does not exist"
+  /Owners/use_cases/{use_case}/upload_and_train:
+    post:
+      security:
+        - JwtAdmin: []
+      operationId: "routes.owners.upload_and_train"
+      tags:
+        - "Owners"
+      summary: "Upload the files required for the federated training"
+      description: "Upload the files required for the federated training"
+      consumes:
+        - multipart/form-data
+      parameters:
+        - name: "use_case"
+          in: "path"
+          description: "Name of the Use-Case to upload to (must already exist)"
+          required: true
+          type: "string"
+        - name: "global_hyperparameters"
+          in: "formData"
+          description: "File containing the global hyperparameters"
+          required: true
+          type: "file"
+        - name: "preprocessing"
+          in: "formData"
+          description: "File containing the preprocessing"
+          required: true
+          type: "file"
+        - name: "model"
+          in: "formData"
+          description: "File containing the keras model"
+          required: true
+          type: "file"
+        - name: "dataset_file1"
+          in: "formData"
+          description: "File1 of the dataset. Functionality is use_case dependendent. (i.e. True Data)"
+          required: true
+          type: "file"
+        - name: "dataset_file2"
+          in: "formData"
+          description: "File2 of the dataset. Functionality is use_case dependendent. (i.e. False Data)"
+          required: true
+          type: "file"
+        #- in: body
+        #  name: "Object"
+        #  required: true
+        #  schema:
+        #    $ref: '#/definitions/LayerMapping'
+      responses:
+        '200':
+          description: "Successful Request"
+        '404':
+          description: "Use case does not exist"
+        '400':
+          description: "Field in request is missing"
+          
+#####
+# Developers
+#####        
+  /Developers/use_case/{use_case}/last_train:
+    get:
+      security:
+        - JwtRegular: []
+      operationId: "routes.developers.last"
+      tags:
+        - "Developers"
+      summary: "Get last train session data"
+      description: "Get last train session data"
+      parameters:
+        - name: "use_case"
+          in: "path"
+          description: "Name of the Use-Case to get the last training session from (must already exist)"
+          required: true
+          type: "string"
+      responses:
+        '200':
+          description: "Successful Request"
+        '404':
+          description: "Use case train session data does not exist"
+  /Developers/use_cases/{use_case}/developer_id/{developer_id}/upload_and_train:
+    post:
+      security:
+        - JwtAdmin: []
+      operationId: "routes.developers.upload_and_train"
+      tags:
+        - "Developers"
+      summary: "Upload the files required and starts the federated training"
+      description: "Upload the files required and starts the federated training"
+      consumes:
+        - multipart/form-data
+      parameters:
+        - name: "use_case"
+          in: "path"
+          description: "Name of the Use-Case to upload to (must already exist)"
+          required: true
+          type: "string"
+        - name: "developer_id"
+          in: "path"
+          description: "Id used to uniquely identify the developer who starts the training"
+          required: true
+          type: "integer"
+        - name: "dataset_file1"
+          in: "formData"
+          description: "File1 of the dataset. Functionality is use_case dependendent. (i.e. True Data)"
+          required: true
+          type: "file"
+        - name: "dataset_file2"
+          in: "formData"
+          description: "File2 of the dataset. Functionality is use_case dependendent. (i.e. False Data)"
+          required: true
+          type: "file"
+        #- in: body
+        #  name: "Object"
+        #  required: true
+        #  schema:
+        #    $ref: '#/definitions/LayerMapping'
+      responses:
+        '200':
+          description: "Successful Request"
+        '404':
+          description: "Use case does not exist"
+        '400':
+          description: "Field in request is missing"
+#####
+# Users
+#####        
+  /Users/use_case/{use_case}/data_entry/{data_entry}/check_article:
+    post:
+      security:
+        - JwtAdmin: []
+      operationId: "routes.users.check_article"
+      tags:
+        - "Users"
+      summary: "Use the trained model to evaluate an input"
+      description: "Use the trained model to evaluate an input"
+      parameters:
+        - name: "use_case"
+          in: "path"
+          description: "Name of the Use-Case to upload to (must already exist)"
+          required: true
+          type: "string"
+        - in: "path"
+          name: "data_entry"
+          required: true
+          type: "string"
+          description: "Data to be verified/parsed by the trained model. I.e: ''Santa Claus is actually real. A new study revealed...'' etc. "
+      responses:
+        '200':
+          description: "Successful Request"
+        '404':
+          description: "Use case does not exist"
+        '400':
+          description: "Field in request is missing"
+          
+    
+
+# definitions:
+#   Data_entry:
+#     type: "object"
+#     required:
+#       - internal
+#     properties:
+#       internal:
+#         type: string
+#         example: "todo, create the preprocessing part"
\ No newline at end of file

src/participation-hub/federated-learning-microservice/app/configs/swagger.yml

+swagger: "2.0"
+info:
+  title: Federated Learning microservice
+  description: This is the documentation for the federated learning microservice.
+  version: "1.0.0"
+
+consumes:
+  - "application/json"
+produces:
+  - "application/json"
+
+basePath: "/api"
+
+# Import security definitions from seperate file
+securityDefinitions:
+  $ref: '../security/security.yml#securityDefinitions'
+
+paths:
+  $ref: 'routes.yml#paths'
\ No newline at end of file

src/participation-hub/federated-learning-microservice/app/configs/swagger_local.yml

+swagger: "2.0"
+info:
+  title: Business Logic microservice
+  description: This is the documentation for the business logic microservice.
+  version: "1.0.0"
+
+consumes:
+  - "application/json"
+produces:
+  - "application/json"
+
+basePath: "/api"
+
+# Import security definitions from seperate file
+securityDefinitions:
+  $ref: '../../../../modules/security/security_local.yml#securityDefinitions'
+
+paths:
+  $ref: 'routes.yml#paths'
\ No newline at end of file

src/participation-hub/federated-learning-microservice/app/debug.py

+from flask import request
+
+def echo():
+    return request.json
+
+ 
\ No newline at end of file

src/participation-hub/federated-learning-microservice/app/implementationLog

+server_weights
+<float32[784,10],float32[10]>@SERVER
+mean_client_wieghts
+<float32[784,10],float32[10]>@SERVER
+## Starting...
+## Preprocessing the federated_train_data
+## Declaring the model
+## Declaring the federated algorithm
+tf_dataset_type
+<x=float32[?,784],y=int32[?,1]>*
+model_weights_type
+<float32[784,10],float32[10]>
+server_state type
+<class 'list'>
+<class 'numpy.ndarray'>
+FINISHEEED
+server_state[1]
+[ 5.4024562e-04 -1.6237081e-03  6.2275940e-04  1.4378619e-05
+ -3.4344319e-04  4.4040685e-04 -6.7906491e-05 -3.0773325e-04
+  1.3574951e-04  5.8925571e-04]
+server2_state[1]
+[ 5.8093132e-04 -2.8670396e-05  1.1061553e-04 -1.5197636e-04
+ -4.6668845e-04  2.7149473e-04 -1.8408171e-04  5.8942172e-05
+ -3.8304061e-04  1.9247324e-04]
+merged_state[1]
+[ 5.6058844e-04 -8.2618924e-04  3.6668745e-04 -6.8798872e-05
+ -4.0506583e-04  3.5595079e-04 -1.2599411e-04 -1.2439553e-04
+ -1.2364556e-04  3.9086447e-04]
+## Evaluation of the model
+server_weights
+<float32[784,10],float32[10]>@SERVER
+mean_client_wieghts
+<float32[784,10],float32[10]>@SERVER
+## Starting...
+## Preprocessing the federated_train_data
+## Declaring the model
+## Declaring the federated algorithm
+tf_dataset_type
+<x=float32[?,784],y=int32[?,1]>*
+model_weights_type
+<float32[784,10],float32[10]>
+server_state type
+<class 'list'>
+<class 'numpy.ndarray'>
+FINISHEEED
+server_state[1]
+[ 5.40245732e-04 -1.62370806e-03  6.22759399e-04  1.43785965e-05
+ -3.43443331e-04  4.40406700e-04 -6.79065852e-05 -3.07733397e-04
+  1.35749448e-04  5.89255709e-04]
+server2_state[1]
+[ 5.8093149e-04 -2.8670451e-05  1.1061558e-04 -1.5197645e-04
+ -4.6668845e-04  2.7149462e-04 -1.8408173e-04  5.8942172e-05
+ -3.8304055e-04  1.9247322e-04]
+merged_state[1]
+[ 5.6058861e-04 -8.2618924e-04  3.6668748e-04 -6.8798923e-05
+ -4.0506589e-04  3.5595067e-04 -1.2599415e-04 -1.2439561e-04
+ -1.2364556e-04  3.9086447e-04]
+## Evaluation of the model
+server_weights
+<float32[784,10],float32[10]>@SERVER
+mean_client_wieghts
+<float32[784,10],float32[10]>@SERVER
+server_weights
+<float32[784,10],float32[10]>@SERVER
+mean_client_wieghts
+<float32[784,10],float32[10]>@SERVER
+## Starting...
+## Preprocessing the federated_train_data
+## Declaring the model
+## Declaring the federated algorithm
+tf_dataset_type
+<x=float32[?,784],y=int32[?,1]>*
+model_weights_type
+<float32[784,10],float32[10]>
+server_state type
+<class 'list'>
+<class 'numpy.ndarray'>
+FINISHEEED
+server_state[1]
+[ 5.4024585e-04 -1.6237078e-03  6.2275951e-04  1.4378643e-05
+ -3.4344324e-04  4.4040682e-04 -6.7906469e-05 -3.0773322e-04
+  1.3574972e-04  5.8925588e-04]
+server2_state[1]
+[ 5.80931432e-04 -2.86704162e-05  1.10615605e-04 -1.51976506e-04
+ -4.66688391e-04  2.71494675e-04 -1.84081669e-04  5.89420633e-05
+ -3.83040548e-04  1.92473250e-04]
+merged_state[1]
+[ 5.6058867e-04 -8.2618912e-04  3.6668757e-04 -6.8798930e-05
+ -4.0506583e-04  3.5595073e-04 -1.2599406e-04 -1.2439558e-04
+ -1.2364541e-04  3.9086456e-04]
+## Evaluation of the model

src/participation-hub/federated-learning-microservice/app/implementationScript.sh

+#! /bin/bash
+
+#source venv/bin/activate
+
+python processing/main_processing.py >> implementationLog

src/participation-hub/federated-learning-microservice/app/main.py

+# add modules folder to interpreter path
+import sys
+import os
+from pathlib import Path
+from typing import Dict, Any
+
+modules_path = '../../../modules/'
+if os.path.exists(modules_path):
+    sys.path.insert(1, modules_path)
+
+# load swagger config
+import connexion
+from security import swagger_util
+from env_info import is_running_locally, get_resources_path
+from flask import request
+from flask import redirect
+
+app = connexion.App(__name__, specification_dir='configs/')
+
+@app.route('/', methods=['GET'])
+def api_root():
+    return redirect('/api/ui')
+
+# SSL configuration
+certificate_path = get_resources_path()
+context = (os.path.normpath(f'{certificate_path}/articonf1.crt'), os.path.normpath(f'{certificate_path}/articonf1.key')) # certificate and key files
+
+if is_running_locally():
+    print("Running locally...")
+    app.add_api(swagger_util.get_bundled_specs(Path("configs/swagger_local.yml")),
+                resolver = connexion.RestyResolver("cms_rest_api"))
+else:
+    app.add_api(swagger_util.get_bundled_specs(Path("configs/swagger.yml")),
+                resolver = connexion.RestyResolver("cms_rest_api"))
+
+# start app
+if __name__ == '__main__':
+    app.run(host='0.0.0.0', port=5000, debug=False, ssl_context=context) #<-- IF running locally comment ss_context

src/participation-hub/federated-learning-microservice/app/requirements.txt

+astroid==2.4.2
+attrs==19.3.0
+autopep8==1.5.4
+certifi==2020.6.20
+cffi==1.14.2
+chardet==3.0.4
+click==7.1.2
+clickclick==1.2.2
+colorama==0.4.3
+connexion==2.7.0
+coverage==5.3.1
+cryptography==3.1
+Flask==1.1.2
+idna==2.10
+importlib-metadata==1.7.0
+inflection==0.5.0
+isort==4.3.21
+itsdangerous==1.1.0
+Jinja2==2.11.2
+jsonschema==3.2.0
+lazy-object-proxy==1.4.3
+MarkupSafe==1.1.1
+mccabe==0.6.1
+openapi-spec-validator==0.2.9
+pandas==1.2.4
+prance==0.19.0
+pycodestyle==2.6.0
+pycparser==2.20
+pylint==2.5.3
+pymongo==3.11.0
+pyrsistent==0.16.0
+PyYAML==5.3.1
+requests==2.24.0
+rope==0.17.0
+semver==2.10.2
+six==1.15.0
+swagger-ui-bundle==0.0.8
+tensorflow==2.3.0
+tensorflow-federated==0.17.0
+toml==0.10.1
+typed-ast==1.4.1
+urllib3==1.25.10
+Werkzeug==1.0.1
+wrapt==1.12.1
+zipp==3.1.0
+

src/participation-hub/federated-learning-microservice/app/routes/developers.py

+import json
+import os
+from flask import Response, request
+import requests
+import pandas as pd
+import sys
+import network_constants
+import time
+from os.path import dirname, abspath
+
+modules_path = './'
+if os.path.exists(modules_path):
+    sys.path.insert(1, modules_path)
+
+def last(use_case: str):    
+    #FORWARD TO GPU SERVER WITH IP AND PORT
+
+    url = f'http://{network_constants.FEDERATED_TRAINING_HOSTNAME}:{network_constants.FEDERATED_TRAINING_REST_PORT}/api/Developers/use_case/{use_case}/last_train'
+    
+    response = requests.get(
+            url,
+            verify = False, 
+            proxies = { "http":None, "https":None }
+    )
+    return json.loads(response.text)
+
+
+def upload_and_train(use_case: str, developer_id: int):
+
+
+    #TODO FORWARD FILES, for some reason  the files are received in this microservice (Federated-LEARNING), but after forwarding they are empty in Federated-TRAINING
+    file_dict = request.files
+
+
+    url = f'http://{network_constants.FEDERATED_TRAINING_HOSTNAME}:{network_constants.FEDERATED_TRAINING_REST_PORT}/api/Developers/use_cases/{use_case}/developer_id/{developer_id}/upload_and_train'
+    response = requests.post(
+            url,
+            verify = False, 
+            proxies = { "http":None, "https":None },
+            files= {k: (v.filename, v.stream, v.content_type, v.headers) for k, v in file_dict.items()}
+            #data = data
+    )
+
+    return json.loads(response.text)
+
+
+#upload_and_train("text_processing",1)
+#last("text_processing")
+

src/participation-hub/federated-learning-microservice/app/routes/owners.py

+import json
+import os
+import sys
+import shutil
+from flask import Response, request
+import requests
+import pandas as pd
+import network_constants
+
+def last(use_case: str):    
+    #FORWARD TO GPU SERVER WITH IP AND POR
+    url = f'https://{network_constants.FEDERATED_TRAINING_HOSTNAME}:{network_constants.FEDERATED_TRAINING_REST_PORT}/api/Owners/use_case/{use_case}/last_train:'
+    response = requests.get(
+            url,
+            verify = False, 
+            proxies = { "http":None, "https":None }
+    )
+    return json.loads(response.text)
+
+
+def upload_and_train(use_case: str):
+
+    #FORWARD TO GPU SERVER WITH IP AND PORT
+    #TODO FORWARD FILES, for some reason  the files are received in this microservice (Federated-LEARNING), but after forwarding they are empty in Federated-TRAINING
+    file_dict = request.files
+    
+    url = f'https://{network_constants.FEDERATED_TRAINING_HOSTNAME}:{network_constants.FEDERATED_TRAINING_REST_PORT}/api/Owners/use_cases/{use_case}/upload_and_train:'
+    response = requests.post(
+            url,
+            verify = False, 
+            proxies = { "http":None, "https":None },
+            files= {k: (v.filename, v.stream, v.content_type, v.headers) for k, v in file_dict.items()}
+    )
+
+    return json.loads(response.text)
+
+
+#last("text_processing")
+#upload_and_train("text_processing") #warning it deletes the files

src/participation-hub/federated-learning-microservice/app/routes/users.py

+
+
+from flask import Response, request
+import sys
+import requests
+import network_constants
+import json
+
+def check_article(use_case: str,data_entry: str):
+
+    #FORWARD TO GPU SERVER WITH IP AND PORT
+    url = f'https://{network_constants.FEDERATED_TRAINING_HOSTNAME}:{network_constants.FEDERATED_TRAINING_REST_PORT}/api/Users/use_case/{use_case}/data_entry/{data_entry}/check_article:'
+    #url = "google.com"#API ENDPOINT WITH IP AND PORT OF GPU SERVER
+    response = requests.get(
+            url,
+            verify = False, 
+            proxies = { "http":None, "https":None }
+    )
+    return json.loads(response.text)
+
+    
+
+#check_article("text_processing")
\ No newline at end of file

src/participation-hub/federated-learning-microservice/app/slurm-1791.out

+python: can't open file 'processing/main_processing.py': [Errno 2] No such file or directory

src/participation-hub/federated-learning-microservice/app/tests/manage_sys_paths.py

+# add modules folder to interpreter path
+import sys
+import os
+modules_paths = ['./', '../', '../../../modules/']
+for path in modules_paths:
+    if os.path.exists(path):
+        sys.path.insert(1, path)

src/participation-hub/federated-learning-microservice/deployment/deployment.yml

+apiVersion: v1
+kind: Service
+metadata:
+  name: federated-learning
+spec:
+  type: LoadBalancer
+  selector:
+      app: federated-learning
+  ports:
+    - name: http
+      port: 80
+      targetPort: 5000
+      nodePort: 30422
+      protocol: TCP
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: federated-learning
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: federated-learning
+  template:
+    metadata:
+      labels:
+        app: federated-learning
+    spec:
+      containers:
+      - name: federated-learning
+        image: alexx882/federated-learning-microservice
+        ports:
+        - containerPort: 5000
+        volumeMounts:
+        - mountPath: /srv/articonf
+          name: articonf
+      volumes:
+      - name: articonf
+        hostPath: 
+          path: /srv/articonf
+          type: Directory
+# ---