Federated Training working server on gpu3

src/modules/network_constants.py

@@ -86,29 +86,29 @@ else:
 
 ## Federated Learning
 if server:
-    BUSINESS_LOGIC_HOSTNAME = 'federated-learning'
-    BUSINESS_LOGIC_DB_HOSTNAME = f'{BUSINESS_LOGIC_HOSTNAME}-db'
-    BUSINESS_LOGIC_REST_PORT = 80
-    BUSINESS_LOGIC_DB_PORT = 27017
+    FEDERATED_LEARNING_HOSTNAME = 'federated-learning'
+    #FEDERATED_LEARNING_DB_HOSTNAME = f'{EDERATED_LEARNING_HOSTNAME}-db'
+    FEDERATED_LEARNING_REST_PORT = 80
+    #FEDERATED_LEARNING_DB_PORT = 27017
 else:
-    BUSINESS_LOGIC_HOSTNAME = 'articonf1.itec.aau.at'
-    BUSINESS_LOGIC_DB_HOSTNAME = 'articonf1.itec.aau.at'
-    BUSINESS_LOGIC_REST_PORT = 30422
-    BUSINESS_LOGIC_DB_PORT = 30423
+    FEDERATED_LEARNING_HOSTNAME = 'articonf1.itec.aau.at'
+    #FEDERATED_LEARNING_DB_HOSTNAME = 'articonf1.itec.aau.at'
+    FEDERATED_LEARNING_REST_PORT = 30422
+    #FEDERATED_LEARNING_DB_PORT = 30423
 
 #endregion Participation Hub
 
 #region Federated Training
 ## Federated Training
 if server:
-    BUSINESS_LOGIC_HOSTNAME = 'federated-training'
-    BUSINESS_LOGIC_DB_HOSTNAME = f'{BUSINESS_LOGIC_HOSTNAME}-db'
-    BUSINESS_LOGIC_REST_PORT = 80
-    BUSINESS_LOGIC_DB_PORT = 27017
+    FEDERATED_TRAINING_HOSTNAME = 'gpu3.itec.aau.at'
+    #FEDERATED_TRAINING_DB_HOSTNAME = f'{FEDERATED_TRAINING_HOSTNAME}-db'
+    FEDERATED_TRAINING_REST_PORT = 30424
+    #FEDERATED_TRAINING_DB_PORT = 27017
 else:
-    BUSINESS_LOGIC_HOSTNAME = 'articonf1.itec.aau.at'
-    BUSINESS_LOGIC_DB_HOSTNAME = 'articonf1.itec.aau.at'
-    BUSINESS_LOGIC_REST_PORT = 30424
-    BUSINESS_LOGIC_DB_PORT = 30425
+    FEDERATED_TRAINING_HOSTNAME = 'gpu3.itec.aau.at'
+    #FEDERATED_TRAINING_DB_HOSTNAME = 'articonf1.itec.aau.at'
+    FEDERATED_TRAINING_REST_PORT = 30424
+    #FEDERATED_TRAINING_DB_PORT = 30425
 
 #endregion Federated Training
\ No newline at end of file

src/participation-hub/federated-learning-microservice/README.md

 # Federated Learning Microservice
-The business model microservice serves as an interface for the individual use cases. Here, schema information for all use cases is stored to enable context-aware processing.
+The Federated Learning microservice serves as an interface for the Federated TRAINING microservice which handles all the federated processing in the back-end.
 
 ## Technologies
 - Python 3.x

tools/federated-training/README.md

-# Federated Learning Microservice
-The business model microservice serves as an interface for the individual use cases. Here, schema information for all use cases is stored to enable context-aware processing.
+# Federated Training Microservice
+Federated Training microservice handles the requests from the Federated LEARNING microservice. After the processing is done, a response with the results is sent back to the Federated Learning microservice.
 
 ## Technologies
 - Python 3.x

tools/federated-training/app/configs/routes.yml

-swagger: '2.0'
+swagger: "2.0"
+info:
+  title: Federated Training microservice
+  description: This is the documentation for the federated Training microservice.
+  version: "1.0.0"
+
+consumes:
+  - "application/json"
+produces:
+  - "application/json"
+
+basePath: "/api"
+
 paths:
   #####
   # Owners
   #####
   /Owners/use_case/{use_case}/last_train:
     get:
-      security:
-        - JwtRegular: []
       operationId: "routes.owners.last"
       tags:
         - "Owners"
@@ -25,8 +35,6 @@ paths:
           description: "Use case train session data does not exist"
   /Owners/use_cases/{use_case}/upload_and_train:
     post:
-      security:
-        - JwtAdmin: []
       operationId: "routes.owners.upload_and_train"
       tags:
         - "Owners"
@@ -83,8 +91,6 @@ paths:
 #####        
   /Developers/use_case/{use_case}/last_train:
     get:
-      security:
-        - JwtRegular: []
       operationId: "routes.developers.last"
       tags:
         - "Developers"
@@ -103,8 +109,6 @@ paths:
           description: "Use case train session data does not exist"
   /Developers/use_cases/{use_case}/developer_id/{developer_id}/upload_and_train:
     post:
-      security:
-        - JwtAdmin: []
       operationId: "routes.developers.upload_and_train"
       tags:
         - "Developers"
@@ -150,8 +154,6 @@ paths:
 #####        
   /Users/use_case/{use_case}/data_entry/{data_entry}/check_article:
     post:
-      security:
-        - JwtAdmin: []
       operationId: "routes.users.check_article"
       tags:
         - "Users"

tools/federated-training/app/configs/swagger.yml

-swagger: "2.0"
-info:
-  title: Federated Training microservice
-  description: This is the documentation for the federated Training microservice.
-  version: "1.0.0"
-
-consumes:
-  - "application/json"
-produces:
-  - "application/json"
-
-basePath: "/api"
-
-# Import security definitions from seperate file
-securityDefinitions:
-  $ref: '../security/security.yml#securityDefinitions'
-
-paths:
-  $ref: 'routes.yml#paths'
\ No newline at end of file

tools/federated-training/app/configs/swagger_local.yml

-swagger: "2.0"
-info:
-  title: Federated Training microservice
-  description: This is the documentation for the federated learning microservice.
-  version: "1.0.0"
-
-consumes:
-  - "application/json"
-produces:
-  - "application/json"
-
-basePath: "/api"
-
-# Import security definitions from seperate file
-securityDefinitions:
-  $ref: '../../../../modules/security/security_local.yml#securityDefinitions'
-
-paths:
-  $ref: 'routes.yml#paths'
\ No newline at end of file

tools/federated-training/app/implementationScript.sh

@@ -2,4 +2,4 @@
 
 #source venv/bin/activate
 
-python processing/main_processing.py >> implementationLog
+python processing/text_processing/main_proc.py >> implementationLog

tools/federated-training/app/main.py

@@ -4,14 +4,14 @@ import os
 from pathlib import Path
 from typing import Dict, Any
 
-modules_path = '../../../modules/'
+modules_path = '../../src/modules/'
 if os.path.exists(modules_path):
     sys.path.insert(1, modules_path)
 
 # load swagger config
 import connexion
-from security import swagger_util
-from env_info import is_running_locally, get_resources_path
+#from security import swagger_util
+#from env_info import is_running_locally, get_resources_path
 from flask import request
 from flask import redirect
 
@@ -29,17 +29,14 @@ def api_root():
     return redirect('/api/ui')
 
 # SSL configuration
-certificate_path = get_resources_path()
-context = (os.path.normpath(f'{certificate_path}/articonf1.crt'), os.path.normpath(f'{certificate_path}/articonf1.key')) # certificate and key files
+#certificate_path = get_resources_path()
+#context = (os.path.normpath(f'{certificate_path}/articonf1.crt'), os.path.normpath(f'{certificate_path}/articonf1.key')) # certificate and key files
+
+# load swagger config 
+app = connexion.App(__name__, specification_dir='configs/')
+app.add_api('routes.yml')
 
-if is_running_locally():
-    print("Running locally...")
-    app.add_api(swagger_util.get_bundled_specs(Path("configs/swagger_local.yml")),
-                resolver = connexion.RestyResolver("cms_rest_api"))
-else:
-    app.add_api(swagger_util.get_bundled_specs(Path("configs/swagger.yml")),
-                resolver = connexion.RestyResolver("cms_rest_api"))
 
 # start app
 if __name__ == '__main__':
-    app.run(host='0.0.0.0', port=5000, debug=False, ssl_context=context)
+    app.run(host='0.0.0.0', port=30424, debug=False)

tools/federated-training/app/print_test.py

+print("TEST WORKED!")
\ No newline at end of file

tools/federated-training/app/processing/text_processing/ledger.csv

@@ -11,3 +11,4 @@ Trainer_id,Model_id,Dataset_id,Accuracy,Loss
 1,1624019715,1624019715,1.0,0.0
 1,1624021190,1624021190,0.75,nan
 1,1624284673,1624284673,0.5,nan
+0,1624550528,1624550528,0.75,nan

tools/federated-training/app/routes/developers.py

@@ -55,6 +55,6 @@ def trainMetricsToJSON(last_train_metrics : list):
     metricsDict["Loss"] = last_train_metrics[4]
     return json.dumps(metricsDict)
 
-upload_and_train("text_processing",1)
+#upload_and_train("text_processing",1)
 #last("text_processing")
 

tools/federated-training/app/startup.py

+/home/itec/bogdan/Articonf/smart/tools/federated-training/app/venv/bin/python3 /home/itec/bogdan/Articonf/smart/tools/federated-training/app/print_test.py
\ No newline at end of file

tools/federated-training/app/tests/manage_sys_paths.py

 # add modules folder to interpreter path
 import sys
 import os
-modules_paths = ['./', '../', '../../../modules/']
+modules_paths = ['./', '../', '../../src/modules/']
 for path in modules_paths:
     if os.path.exists(path):
         sys.path.insert(1, path)