Skip to content

S
SMART
Commit 18dde0e8 authored by Bogdan Mihai (ARTICONF student)'s avatar Bogdan Mihai (ARTICONF student)
Browse files
Options

Implemented basic routes functionality

parent 1c2b9a52
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 45898 additions and 1849 deletions
src/participation-hub/federated-learning-microservice/app/mylog deleted 100644 → 0
View file @ 1c2b9a52
Submitted batch job 1721
Submitted batch job 1722
Submitted batch job 1723
Submitted batch job 1724
Submitted batch job 1725
Submitted batch job 1726
Submitted batch job 1727
Submitted batch job 1728
Submitted batch job 1729
Submitted batch job 1730
Submitted batch job 1731
Submitted batch job 1732
Submitted batch job 1733
Submitted batch job 1734
Submitted batch job 1735
Submitted batch job 1736
Submitted batch job 1737
Submitted batch job 1738
Submitted batch job 1739
Submitted batch job 1740
Submitted batch job 1741
Submitted batch job 1742
Submitted batch job 1743
Submitted batch job 1744
Submitted batch job 1745
Submitted batch job 1746
Submitted batch job 1747
Submitted batch job 1748
Submitted batch job 1749
Submitted batch job 1750
Submitted batch job 1751
Submitted batch job 1752
Submitted batch job 1753
Submitted batch job 1754
Submitted batch job 1755
Submitted batch job 1756
Submitted batch job 1757
Submitted batch job 1758
Submitted batch job 1759
Submitted batch job 1760
Submitted batch job 1761
Submitted batch job 1762
Submitted batch job 1763
Submitted batch job 1764
Submitted batch job 1765
Submitted batch job 1766
Submitted batch job 1767
Submitted batch job 1768
Submitted batch job 1769
Submitted batch job 1777
Submitted batch job 1778
Submitted batch job 1779
Submitted batch job 1780
Submitted batch job 1781
Submitted batch job 1782
Submitted batch job 1783
Submitted batch job 1784
Submitted batch job 1785
Submitted batch job 1786
Submitted batch job 1787
Submitted batch job 1788
Submitted batch job 1789
Submitted batch job 1790
Submitted batch job 1791
src/participation-hub/federated-learning-microservice/app/mylogsource deleted 100644 → 0
View file @ 1c2b9a52
Submitted batch job 1770
Submitted batch job 1771
Submitted batch job 1772
Submitted batch job 1773
Submitted batch job 1774
Submitted batch job 1775
Submitted batch job 1776
src/participation-hub/federated-learning-microservice/app/processing/default/federated_algorithm.py 0 → 100644
View file @ 18dde0e8
#import processing.text_processing.global_hyperparams as globals
import global_hyperparams as globals
from model import get_simple_LSTM_model
import pandas as pd
from sklearn.model_selection import train_test_split
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import LSTM, Dense, Dropout, Embedding
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences
import collections
import numpy as np
import tensorflow as tf
import tensorflow_federated as tff
def model_fn():
keras_model = get_simple_LSTM_model()
#return tff.learning.from_compiled_keras_model(keras_model, sample_batch) original
return tff.learning.from_keras_model(
keras_model,
input_spec=globals.INPUT_SPEC,
loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
metrics=[tf.keras.metrics.SparseCategoricalAccuracy()])
def federated_computation_new(train_dataset,test_dataset):
if(globals.INPUT_SPEC == None):
#should never reach this place because INPUT_SPEC is instantiated inside get_preprocessed_train_test_data.
#however, if in the future, processed data is provided without hte preprocessing function it will be none -> therefore assign it here
globals.INPUT_SPEC = train_dataset[0].element_spec
# Training and evaluating the model
iterative_process = tff.learning.build_federated_averaging_process(model_fn,client_optimizer_fn=lambda: tf.keras.optimizers.SGD(lr=0.5))
state = iterative_process.initialize()
print(type(state))
for n in range(globals.EPOCHS):
state, metrics = iterative_process.next(state, train_dataset)
print('round {}, training metrics={}'.format(n+1, metrics))
evaluation = tff.learning.build_federated_evaluation(model_fn)
eval_metrics = evaluation(state.model, train_dataset)
print('Training evaluation metrics={}'.format(eval_metrics))
test_metrics = evaluation(state.model, test_dataset)
print('Test evaluation metrics={}'.format(test_metrics))
return state,metrics
###############################################################################################
def federated_computation_continue(train_dataset,test_dataset,restored_state):
if(globals.INPUT_SPEC == None):
#should never reach this place because INPUT_SPEC is instantiated inside get_preprocessed_train_test_data.
#however, if in the future, processed data is provided without hte preprocessing function it will be none -> therefore assign it here
globals.INPUT_SPEC = train_dataset[0].element_spec
# Training and evaluating the model
iterative_process = tff.learning.build_federated_averaging_process(model_fn,client_optimizer_fn=lambda: tf.keras.optimizers.SGD(lr=0.5))
state = iterative_process.initialize()
state = restored_state
print(type(state))
for n in range(globals.EPOCHS):
state, metrics = iterative_process.next(state, train_dataset)
print('round {}, training metrics={}'.format(n+1, metrics))
evaluation = tff.learning.build_federated_evaluation(model_fn)
eval_metrics = evaluation(state.model, train_dataset)
print('Training evaluation metrics={}'.format(eval_metrics))
test_metrics = evaluation(state.model, test_dataset)
print('Test evaluation metrics={}'.format(test_metrics))
return state,metrics
###############################################################################################
src/participation-hub/federated-learning-microservice/app/processing/default/main_proc.py 0 → 100644
View file @ 18dde0e8
import os
#from processing.text_processing.federated_algorithm import federated_computation_continue
#from processing.text_processing.version_handler import save_state_to_file
print(os.getcwd())
#import processing.text_processing.global_hyperparams as globals
#from processing.text_processing.preprocessing import get_preprocessed_train_test_data
import global_hyperparams as globals
from preprocessing import get_preprocessed_train_test_data
from federated_algorithm import federated_computation_new, federated_computation_continue, save_state_to_file, load_state_from_file
from checkpoint_manager import save_to_file_CSV#,save_state_to_file, load_state_from_file
# globals.initialize()
# train_dataset, test_dataset= get_preprocessed_train_test_data()
# state,metrics = federated_computation_new(train_dataset,test_dataset)
# last_model_id = save_state_to_file(state)
# #model_filename = "ckpt_1622721644"
# #restored_state = load_state_from_file(model_filename)
# #state,metrics = federated_computation_continue(train_dataset, test_dataset, restored_state)
# #last_model_id = save_state_to_file(state)
# trained_metrics= metrics['train']
# save_to_file_CSV(globals.TRAINER_ID,last_model_id,globals.DATASET_ID,trained_metrics['sparse_categorical_accuracy'],trained_metrics['loss'])
# print("DONE")
# print(type(state))
# print(type(metrics))
# print("DONE2")
def start_processing(developer_id:int = 0):
globals.initialize()
train_dataset, test_dataset= get_preprocessed_train_test_data()
state,metrics = federated_computation_new(train_dataset,test_dataset)
trained_metrics= metrics['train']
timestamp = save_state_to_file(state)
globals.TRAINER_ID = developer_id
globals.DATASET_ID = timestamp
written_row = save_to_file_CSV(globals.TRAINER_ID,timestamp,globals.DATASET_ID,trained_metrics['sparse_categorical_accuracy'],trained_metrics['loss'])
return written_row
\ No newline at end of file
src/participation-hub/federated-learning-microservice/app/processing/prototype/evaluation.py deleted 100644 → 0
View file @ 1c2b9a52
import numpy as np
import tensorflow as tf
import tensorflow_federated as tff
#from models import MnistModel
from models2 import model_fn
def evaluate(server_state):
keras_model = model_fn
keras_model.compile(
loss=tf.keras.losses.SparseCategoricalCrossentropy(),
metrics=[tf.keras.metrics.SparseCategoricalAccuracy()]
)
keras_model.set_weights(server_state)
keras_model.evaluate(central_emnist_test)
def evaluate2(server_state,tff_learning_model):
#TODO: Assign weights to the model
#First idea = server_update function??
evaluation = tff.learning.build_federated_evaluation(tff_learning_model)
# keras_model = MnistModel()
# keras_model.compile(
# loss=tf.keras.losses.SparseCategoricalCrossentropy(),
# metrics=[tf.keras.metrics.SparseCategoricalAccuracy()]
# )
# keras_model.set_weights(server_state)
# keras_model.evaluate(central_emnist_test)
src/participation-hub/federated-learning-microservice/app/processing/prototype/federated_training_algorithm copy.py deleted 100644 → 0
View file @ 1c2b9a52
import numpy as np
import tensorflow as tf
import tensorflow_federated as tff
from models import MnistModel
from models2 import model_fn
#tf_dataset_type = None
#model_weights_type = None
@tf.function
def client_update(model, dataset, server_weights, client_optimizer):
"""Performs training (using the server model weights) on the client's dataset."""
# Initialize the client model with the current server weights.
client_weights = model.trainable_variables
# Assign the server weights to the client model.
tf.nest.map_structure(lambda x, y: x.assign(y),
client_weights, server_weights)
# Use the client_optimizer to update the local model.
for batch in dataset:
with tf.GradientTape() as tape:
# Compute a forward pass on the batch of data
outputs = model.forward_pass(batch)
# Compute the corresponding gradient
grads = tape.gradient(outputs.loss, client_weights)
grads_and_vars = zip(grads, client_weights)
# Apply the gradient using a client optimizer.
client_optimizer.apply_gradients(grads_and_vars)
return client_weights
@tf.function
def server_update(model, mean_client_weights):
"""Updates the server model weights as the average of the client model weights."""
model_weights = model.trainable_variables
# Assign the mean client weights to the server model.
tf.nest.map_structure(lambda x, y: x.assign(y),
model_weights, mean_client_weights)
return model_weights
#Creating the initialization computation
@tff.tf_computation
def server_init():
model = MnistModel() #model_fn()
return model.trainable_variables
@tff.federated_computation
def initialize_fn():
return tff.federated_value(server_init(), tff.SERVER)
whimsy_model = MnistModel()
tf_dataset_type = tff.SequenceType(whimsy_model.input_spec)
model_weights_type = server_init.type_signature.result
@tff.tf_computation(tf_dataset_type, model_weights_type)
def client_update_fn(tf_dataset, server_weights):
model = MnistModel()#model_fn()
client_optimizer = tf.keras.optimizers.SGD(learning_rate=0.01)
return client_update(model, tf_dataset, server_weights, client_optimizer)
@tff.tf_computation(model_weights_type)
def server_update_fn(mean_client_weights):
model = MnistModel()#model_fn()
return server_update(model, mean_client_weights)
federated_server_type = tff.FederatedType(model_weights_type, tff.SERVER)
federated_dataset_type = tff.FederatedType(tf_dataset_type, tff.CLIENTS)
@tff.federated_computation(federated_server_type, federated_dataset_type)
def next_fn(server_weights, federated_dataset):
# Broadcast the server weights to the clients.
print("server_weights")
print(str(server_weights.type_signature))
server_weights_at_client = tff.federated_broadcast(server_weights)
# Each client computes their updated weights.
client_weights = tff.federated_map(
client_update_fn, (federated_dataset, server_weights_at_client))
# The server averages these updates.
mean_client_weights = tff.federated_mean(client_weights)
print("mean_client_wieghts")
print(str(mean_client_weights.type_signature))
# The server updates its model.
server_weights = tff.federated_map(server_update_fn, mean_client_weights)
return server_weights
def get_federated_algorithm():
#Creating the next_fn
#Getting the data type, needed explicitily in the modell functions
whimsy_model = MnistModel() #model_fn()
global tf_dataset_type
tf_dataset_type = tff.SequenceType(whimsy_model.input_spec)
print("tf_dataset_type")
print(str(tf_dataset_type))
global model_weights_type
model_weights_type = server_init.type_signature.result
print("model_weights_type")
print(str(model_weights_type))
# finished printing types
federated_server_type = tff.FederatedType(model_weights_type, tff.SERVER)
federated_dataset_type = tff.FederatedType(tf_dataset_type, tff.CLIENTS)
federated_algorithm = tff.templates.IterativeProcess(
initialize_fn=initialize_fn,
next_fn=next_fn
)
return federated_algorithm
def merge_2_states(server_state1, server_state2):
return np.mean( np.array([ server_state, server2_state ]), axis=0 )
\ No newline at end of file
src/participation-hub/federated-learning-microservice/app/processing/prototype/federated_training_algorithm.py deleted 100644 → 0
View file @ 1c2b9a52
import numpy as np
import tensorflow as tf
import tensorflow_federated as tff
#from models import MnistModel
from models2 import model_fn
#tf_dataset_type = None
#model_weights_type = None
@tf.function
def client_update(model, dataset, server_weights, client_optimizer):
"""Performs training (using the server model weights) on the client's dataset."""
# Initialize the client model with the current server weights.
client_weights = model.trainable_variables
# Assign the server weights to the client model.
tf.nest.map_structure(lambda x, y: x.assign(y),
client_weights, server_weights)
# Use the client_optimizer to update the local model.
for batch in dataset:
with tf.GradientTape() as tape:
# Compute a forward pass on the batch of data
outputs = model.forward_pass(batch)
# Compute the corresponding gradient
grads = tape.gradient(outputs.loss, client_weights)
grads_and_vars = zip(grads, client_weights)
# Apply the gradient using a client optimizer.
client_optimizer.apply_gradients(grads_and_vars)
return client_weights
@tf.function
def server_update(model, mean_client_weights):
"""Updates the server model weights as the average of the client model weights."""
model_weights = model.trainable_variables
# Assign the mean client weights to the server model.
tf.nest.map_structure(lambda x, y: x.assign(y),
model_weights, mean_client_weights)
return model_weights
#Creating the initialization computation
@tff.tf_computation
def server_init():
model = model_fn()
return model.trainable_variables
@tff.federated_computation
def initialize_fn():
return tff.federated_value(server_init(), tff.SERVER)
whimsy_model = model_fn()
tf_dataset_type = tff.SequenceType(whimsy_model.input_spec)
model_weights_type = server_init.type_signature.result
@tff.tf_computation(tf_dataset_type, model_weights_type)
def client_update_fn(tf_dataset, server_weights):
model = model_fn()
client_optimizer = tf.keras.optimizers.SGD(learning_rate=0.01)
return client_update(model, tf_dataset, server_weights, client_optimizer)
@tff.tf_computation(model_weights_type)
def server_update_fn(mean_client_weights):
model = model_fn()
return server_update(model, mean_client_weights)
federated_server_type = tff.FederatedType(model_weights_type, tff.SERVER)
federated_dataset_type = tff.FederatedType(tf_dataset_type, tff.CLIENTS)
@tff.federated_computation(federated_server_type, federated_dataset_type)
def next_fn(server_weights, federated_dataset):
# Broadcast the server weights to the clients.
print("server_weights")
print(str(server_weights.type_signature))
server_weights_at_client = tff.federated_broadcast(server_weights)
# Each client computes their updated weights.
client_weights = tff.federated_map(
client_update_fn, (federated_dataset, server_weights_at_client))
# The server averages these updates.
mean_client_weights = tff.federated_mean(client_weights)
print("mean_client_wieghts")
print(str(mean_client_weights.type_signature))
# The server updates its model.
server_weights = tff.federated_map(server_update_fn, mean_client_weights)
return server_weights
def get_federated_algorithm():
#Creating the next_fn
#Getting the data type, needed explicitily in the modell functions
whimsy_model = model_fn()
global tf_dataset_type
tf_dataset_type = tff.SequenceType(whimsy_model.input_spec)
print("tf_dataset_type")
print(str(tf_dataset_type))
global model_weights_type
model_weights_type = server_init.type_signature.result
print("model_weights_type")
print(str(model_weights_type))
# finished printing types
federated_server_type = tff.FederatedType(model_weights_type, tff.SERVER)
federated_dataset_type = tff.FederatedType(tf_dataset_type, tff.CLIENTS)
federated_algorithm = tff.templates.IterativeProcess(
initialize_fn=initialize_fn,
next_fn=next_fn
)
return federated_algorithm
def merge_2_states(server1_state, server2_state):
return np.mean( np.array([ server1_state, server2_state ]), axis=0 )
\ No newline at end of file
src/participation-hub/federated-learning-microservice/app/processing/prototype/main_processing.py deleted 100644 → 0
View file @ 1c2b9a52
import collections
import attr
import functools
import numpy as np
import tensorflow as tf
import tensorflow_federated as tff
from preprocessing import get_client_ids
from preprocessing import get_federated_train_data
from preprocessing import preprocess
#from models import MnistModel
from federated_training_algorithm import get_federated_algorithm
from federated_training_algorithm import merge_2_states
from evaluation import evaluate
np.random.seed(0)
print("## Starting...")
#GET THE DATA (for now it's the default dataset)
emnist_train, emnist_test = tff.simulation.datasets.emnist.load_data()
######client_ids = get_client_ids(emnist_train) not used
print("## Preprocessing the federated_train_data")
federated_train_data = get_federated_train_data(emnist_train)
print("## Declaring the model")
#it is done in models.py
print("## Declaring the federated algorithm")
federated_algorithm = get_federated_algorithm()
server_state = federated_algorithm.initialize()
for round in range(20):
server_state = federated_algorithm.next(server_state, federated_train_data)
print("server_state type")
print(str(type(server_state)))
print(str(type(server_state[0])))
print("FINISHEEED")
server2_state = federated_algorithm.initialize()
for round in range(2):
server2_state = federated_algorithm.next(server2_state, federated_train_data)
merged_state = merge_2_states(server_state,server2_state)
print("server_state[1]")
print(server_state[1])
print("server2_state[1]")
print(server2_state[1])
print("merged_state[1]")
print(merged_state[1])
# print("federated_algorithm.initialize.type_signature")
# print(str(federated_algorithm.initialize.type_signature)
# print("federated_algorithm.next.type_signature")
# print(str(federated_algorithm.next.type_signature))
# print("## Training the model")
# iterative_process = tff.learning.build_federated_averaging_process(
# MnistModel,
# client_optimizer_fn=lambda: tf.keras.optimizers.SGD(learning_rate=0.02))
# state = iterative_process.initialize()
# for round_num in range(1, 11):
# state, metrics = iterative_process.next(state, federated_train_data)
# print('round {:2d}, metrics={}'.format(round_num, metrics))
#evaluation = tff.learning.build_federated_evaluation(MnistModel)
#TODO integration
print("## Evaluation of the model")
\ No newline at end of file
src/participation-hub/federated-learning-microservice/app/processing/prototype/model_mnist_OLD.py deleted 100644 → 0
View file @ 1c2b9a52
import numpy as np
import tensorflow as tf
import tensorflow_federated as tff
import collections
input_spec_data_global = None
MnistVariables = collections.namedtuple(
'MnistVariables', 'weights bias num_examples loss_sum accuracy_sum')
def create_mnist_variables():
return MnistVariables(
weights=tf.Variable(
lambda: tf.zeros(dtype=tf.float32, shape=(784, 10)),
name='weights',
trainable=True),
bias=tf.Variable(
lambda: tf.zeros(dtype=tf.float32, shape=(10)),
name='bias',
trainable=True),
num_examples=tf.Variable(0.0, name='num_examples', trainable=False),
loss_sum=tf.Variable(0.0, name='loss_sum', trainable=False),
accuracy_sum=tf.Variable(0.0, name='accuracy_sum', trainable=False))
def mnist_forward_pass(variables, batch):
y = tf.nn.softmax(tf.matmul(batch['x'], variables.weights) + variables.bias)
predictions = tf.cast(tf.argmax(y, 1), tf.int32)
flat_labels = tf.reshape(batch['y'], [-1])
loss = -tf.reduce_mean(
tf.reduce_sum(tf.one_hot(flat_labels, 10) * tf.math.log(y), axis=[1]))
accuracy = tf.reduce_mean(
tf.cast(tf.equal(predictions, flat_labels), tf.float32))
num_examples = tf.cast(tf.size(batch['y']), tf.float32)
variables.num_examples.assign_add(num_examples)
variables.loss_sum.assign_add(loss * num_examples)
variables.accuracy_sum.assign_add(accuracy * num_examples)
return loss, predictions
def get_local_mnist_metrics(variables):
return collections.OrderedDict(
num_examples=variables.num_examples,
loss=variables.loss_sum / variables.num_examples,
accuracy=variables.accuracy_sum / variables.num_examples)
@tff.federated_computation
def aggregate_mnist_metrics_across_clients(metrics):
return collections.OrderedDict(
num_examples=tff.federated_sum(metrics.num_examples),
loss=tff.federated_mean(metrics.loss, metrics.num_examples),
accuracy=tff.federated_mean(metrics.accuracy, metrics.num_examples))
class MnistModel(tff.learning.Model):
def __init__(self):
self._variables = create_mnist_variables()
@property
def trainable_variables(self):
return [self._variables.weights, self._variables.bias]
@property
def non_trainable_variables(self):
return []
@property
def local_variables(self):
return [
self._variables.num_examples, self._variables.loss_sum,
self._variables.accuracy_sum
]
@property
def input_spec(self):
return collections.OrderedDict(
x=tf.TensorSpec([None, 784], tf.float32),
y=tf.TensorSpec([None, 1], tf.int32))
@tf.function
def forward_pass(self, batch, training=True):
del training
loss, predictions = mnist_forward_pass(self._variables, batch)
num_exmaples = tf.shape(batch['x'])[0]
return tff.learning.BatchOutput(
loss=loss, predictions=predictions, num_examples=num_exmaples)
@tf.function
def report_local_outputs(self):
return get_local_mnist_metrics(self._variables)
@property
def federated_output_computation(self):
return aggregate_mnist_metrics_across_clients
\ No newline at end of file
src/participation-hub/federated-learning-microservice/app/processing/prototype/models2.py deleted 100644 → 0
View file @ 1c2b9a52
import numpy as np
import tensorflow as tf
import tensorflow_federated as tff
import collections
def create_keras_model(): #### DEFAULT TEST MODEL
return tf.keras.models.Sequential([
tf.keras.layers.InputLayer(input_shape=(784,)),
tf.keras.layers.Dense(10, kernel_initializer='zeros'),
tf.keras.layers.Softmax(),
])
def model_fn():
# We _must_ create a new model here, and _not_ capture it from an external
# scope. TFF will call this within different graph contexts.
keras_model = create_keras_model() ###TODO CAN BE CHANGED TO INCLUDE CALLS TO OTHER MODELS (or it cannot, because you cannot specify which model you want to call?)
return tff.learning.from_keras_model(
keras_model,
input_spec=get_input_spec(),
loss=tf.keras.losses.SparseCategoricalCrossentropy(),
metrics=[tf.keras.metrics.SparseCategoricalAccuracy()])
def get_input_spec():
return collections.OrderedDict(
x=tf.TensorSpec([None, 784], tf.float32),
y=tf.TensorSpec([None, 1], tf.int32))
\ No newline at end of file
src/participation-hub/federated-learning-microservice/app/processing/prototype/preprocessing.py deleted 100644 → 0
View file @ 1c2b9a52
import collections
import attr
import functools
import numpy as np
import tensorflow as tf
import tensorflow_federated as tff
np.random.seed(0)
emnist_train, emnist_test = tff.simulation.datasets.emnist.load_data()
# NUM_CLIENTS = 10
# BATCH_SIZE = 20
def preprocess(dataset, BATCH_SIZE = 20):
def batch_format_fn(element):
"""Flatten a batch of EMNIST data and return a (features, label) tuple."""
return (tf.reshape(element['pixels'], [-1, 784]),
tf.reshape(element['label'], [-1, 1]))
return dataset.batch(BATCH_SIZE).map(batch_format_fn)
def get_client_ids(emnist_train, NUM_CLIENTS = 10):
return np.random.choice(emnist_train.client_ids, size=NUM_CLIENTS, replace=False)
#client_ids = np.random.choice(emnist_train.client_ids, size=NUM_CLIENTS, replace=False)
def get_federated_train_data(emnist_train, NUM_CLIENTS = 10):
client_ids = get_client_ids(emnist_train, NUM_CLIENTS)
return [preprocess(emnist_train.create_tf_dataset_for_client(x))
for x in client_ids
]
# federated_train_data = [preprocess(emnist_train.create_tf_dataset_for_client(x))
# for x in client_ids
# ]
##################################################################
## Second Dataset
############################################
def preprocess_and_shuffle(dataset):
"""Applies `preprocess_dataset` above and shuffles the result."""
preprocessed = preprocess(dataset)
return preprocessed.shuffle(buffer_size=5)
\ No newline at end of file
src/participation-hub/federated-learning-microservice/app/processing/text_processing/federated_algorithm.py
View file @ 18dde0e8
......@@ -26,7 +26,7 @@ def model_fn():
loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
metrics=[tf.keras.metrics.SparseCategoricalAccuracy()])
def federated_computation(train_dataset,test_dataset):
def federated_computation_new(train_dataset,test_dataset):
if(globals.INPUT_SPEC == None):
#should never reach this place because INPUT_SPEC is instantiated inside get_preprocessed_train_test_data.
......@@ -37,6 +37,36 @@ def federated_computation(train_dataset,test_dataset):
iterative_process = tff.learning.build_federated_averaging_process(model_fn,client_optimizer_fn=lambda: tf.keras.optimizers.SGD(lr=0.5))
state = iterative_process.initialize()
print(type(state))
for n in range(globals.EPOCHS):
state, metrics = iterative_process.next(state, train_dataset)
print('round {}, training metrics={}'.format(n+1, metrics))
evaluation = tff.learning.build_federated_evaluation(model_fn)
eval_metrics = evaluation(state.model, train_dataset)
print('Training evaluation metrics={}'.format(eval_metrics))
test_metrics = evaluation(state.model, test_dataset)
print('Test evaluation metrics={}'.format(test_metrics))
return state,metrics
###############################################################################################
def federated_computation_continue(train_dataset,test_dataset,restored_state):
if(globals.INPUT_SPEC == None):
#should never reach this place because INPUT_SPEC is instantiated inside get_preprocessed_train_test_data.
#however, if in the future, processed data is provided without hte preprocessing function it will be none -> therefore assign it here
globals.INPUT_SPEC = train_dataset[0].element_spec
# Training and evaluating the model
iterative_process = tff.learning.build_federated_averaging_process(model_fn,client_optimizer_fn=lambda: tf.keras.optimizers.SGD(lr=0.5))
state = iterative_process.initialize()
state = restored_state
print(type(state))
for n in range(globals.EPOCHS):
state, metrics = iterative_process.next(state, train_dataset)
print('round {}, training metrics={}'.format(n+1, metrics))
......@@ -47,3 +77,6 @@ def federated_computation(train_dataset,test_dataset):
test_metrics = evaluation(state.model, test_dataset)
print('Test evaluation metrics={}'.format(test_metrics))
return state,metrics
###############################################################################################
src/participation-hub/federated-learning-microservice/app/processing/text_processing/global_hyperparams.py
View file @ 18dde0e8
......@@ -18,4 +18,10 @@ def initialize():
global LSTM_OUT # output size of the LSTM layer
LSTM_OUT = 100
global EPOCHS #number of epochs the model will be trained
EPOCHS = 5
\ No newline at end of file
EPOCHS = 5
global TRAINER_ID # ID of the trainer entity.
TRAINER_ID = 0 #0 = Owner of the use_case
global DATASET_ID # ID of the dataset used
DATASET_ID = 0 #0 = "Main"/Original dataset
global USE_CASE #Use_case name
USE_CASE = None
\ No newline at end of file
src/participation-hub/federated-learning-microservice/app/processing/text_processing/ledger.csv 0 → 100644
View file @ 18dde0e8
Trainer_id,Model_id,Dataset_id,Accuracy,Loss
0,1623160388,0,0.25,nan
0,1623160474,0,0.5,nan
1,1623333361,0,0.5,nan
1,1623406445,0,0.5,nan
0,1623419415,1623419415,0.0,nan
src/participation-hub/federated-learning-microservice/app/processing/text_processing/main_proc.py
View file @ 18dde0e8
import os
print(os.getcwd())
#from processing.text_processing.federated_algorithm import federated_computation_continue
#from processing.text_processing.version_handler import save_state_to_file
#import processing.text_processing.global_hyperparams as globals
......@@ -8,12 +8,44 @@ print(os.getcwd())
import global_hyperparams as globals
from preprocessing import get_preprocessed_train_test_data
from federated_algorithm import federated_computation
from federated_algorithm import federated_computation_new, federated_computation_continue
from checkpoint_manager import save_to_file_CSV, save_state_to_file, load_state_from_file
if __name__ == "__main__":
# globals.initialize()
# train_dataset, test_dataset= get_preprocessed_train_test_data()
# state,metrics = federated_computation_new(train_dataset,test_dataset)
# last_model_id = save_state_to_file(state)
# #model_filename = "ckpt_1622721644"
# #restored_state = load_state_from_file(model_filename)
# #state,metrics = federated_computation_continue(train_dataset, test_dataset, restored_state)
# #last_model_id = save_state_to_file(state)
# trained_metrics= metrics['train']
# save_to_file_CSV(globals.TRAINER_ID,last_model_id,globals.DATASET_ID,trained_metrics['sparse_categorical_accuracy'],trained_metrics['loss'])
# print("DONE")
# print(type(state))
# print(type(metrics))
# print("DONE2")
def start_processing(use_case:str,developer_id:int = 0):
globals.initialize()
globals.USE_CASE = use_case
train_dataset, test_dataset= get_preprocessed_train_test_data()
federated_computation(train_dataset,test_dataset)
print("DONE")
state,metrics = federated_computation_new(train_dataset,test_dataset)
trained_metrics= metrics['train']
timestamp = save_state_to_file(state)
globals.TRAINER_ID = developer_id
globals.DATASET_ID = timestamp
written_row = save_to_file_CSV(globals.TRAINER_ID,timestamp,globals.DATASET_ID,trained_metrics['sparse_categorical_accuracy'],trained_metrics['loss'])
return written_row
start_processing("text_processing")
\ No newline at end of file
src/participation-hub/federated-learning-microservice/app/processing/text_processing/preprocessing.py
View file @ 18dde0e8
......@@ -12,6 +12,7 @@ import collections
import numpy as np
import tensorflow as tf
import tensorflow_federated as tff
import os
# real = pd.read_csv("processing/fake_news/prototype_db_fake_real/True.csv")
# fake = pd.read_csv("processing/fake_news/prototype_db_fake_real/Fake.csv")
......@@ -19,8 +20,9 @@ import tensorflow_federated as tff
def get_raw_data()-> tuple:
real = pd.read_csv("processing/fake_news/prototype_db_fake_real/True.csv")
fake = pd.read_csv("processing/fake_news/prototype_db_fake_real/Fake.csv")
real = pd.read_csv("processing/"+globals.USE_CASE+"/db/True.csv")
fake = pd.read_csv("processing/"+globals.USE_CASE+"/db/Fake.csv")
return real,fake
......
src/participation-hub/federated-learning-microservice/app/routes/developers.py 0 → 100644
View file @ 18dde0e8
import json
import os
from flask import Response, request
import pandas as pd
import sys
modules_path = './'
if os.path.exists(modules_path):
sys.path.insert(1, modules_path)
def last(use_case: str):
csv_path = './processing/'+use_case+'/ledger.csv'
try:
df = pd.read_csv(csv_path)
bottom = df.tail(1)
bottom = str(bottom)
print(bottom)
return Response(status=200, response=bottom)
except Exception as e:
print(e)
return Response(status=400, response="Trained model data doesn't exist")
def upload_and_train(use_case: str, developer_id: int):
use_case_path = 'processing/'+use_case+'/'
sys.path.append(use_case_path)
import main_proc
#COPY THE NEW DB TO THE FOLDER
#TODO IMPLEMENT HERE
#THEN start processing
last_train_metrics = main_proc.start_processing(use_case,developer_id)
print (last_train_metrics)
return Response(status=200, response=last_train_metrics)
upload_and_train("text_processing",1)
\ No newline at end of file
src/participation-hub/federated-learning-microservice/app/routes/owners.py 0 → 100644
View file @ 18dde0e8
import json
import os
import sys
import shutil
from flask import Response, request
import pandas as pd
def last(use_case: str):
csv_path = './processing/'+use_case+'/ledger.csv'
try:
df = pd.read_csv(csv_path)
bottom = df.tail(1)
bottom = str(bottom)
print(bottom)
return Response(status=200, response=bottom)
except Exception as e:
print(e)
return Response(status=400, response="Trained model data doesn't exist")
def upload_and_train(use_case: str):
use_case_path = './processing/'+use_case
#Remove old files
try:
if os.path.exists(use_case_path):
print("Use_case path")
print(use_case_path)
shutil.rmtree(use_case_path)
except OSError as error:
print(error)
return Response(status=400, response="Error occured when deleteing the old use_case directory")
#Start a new implementation of the model.
#TODO: get the python files and create them locally in the {use_case} folder
try:
os.mkdir(use_case_path)
#COPY DEFAULT FILES
default_path = 'processing/default/'
use_case_path +='/'
shutil.copyfile(default_path+'main_proc.py',use_case_path+'main_proc.py')
shutil.copyfile(default_path+'__init__.py',use_case_path+'__init__.py')
shutil.copyfile(default_path+'checkpoint_manager.py',use_case_path+'checkpoint_manager.py')
shutil.copyfile(default_path+'federated_algorithm.py',use_case_path+'federated_algorithm.py')
#COPY flask files
#use flask? request.files.getlist('filename')[0] ???
except OSError as error:
print(error)
return Response(status=400, response="Error occured when creating/copying the use_case files")
#TODO: after the files are copied, start training
use_case_path = 'processing/'+use_case+'/'
sys.path.append(use_case_path)
import main_proc
last_train_metrics = main_proc.start_processing(use_case,0)
print (last_train_metrics)
return Response(status=200, response=last_train_metrics)
last("text_processing")
upload_and_train("test")
\ No newline at end of file
src/participation-hub/federated-learning-microservice/app/routes/users.py 0 → 100644
View file @ 18dde0e8
from flask import Response, request
def check_article(use_case: str):
#body = request.STRING
#FOR USE_CASE {use_case}
#insert body into the trained model
#get the result
result = None #bool True/False
return Response(status=400, response=str(result))
\ No newline at end of file
src/participation-hub/federated-learning-microservice/app/slurm-1779.out deleted 100644 → 0
View file @ 1c2b9a52
2021-05-04 14:08:53.127296: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
2021-05-04 14:09:04.981860: W tensorflow/stream_executor/platform/default/dso_loader.cc:59] Could not load dynamic library 'libcuda.so.1'; dlerror: libcuda.so.1: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64
2021-05-04 14:09:04.982134: W tensorflow/stream_executor/cuda/cuda_driver.cc:312] failed call to cuInit: UNKNOWN ERROR (303)
2021-05-04 14:09:04.982158: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:156] kernel driver does not appear to be running on this host (mcore2): /proc/driver/nvidia/version does not exist
2021-05-04 14:09:04.982428: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN)to use the following CPU instructions in performance-critical operations: AVX2 FMA
To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
2021-05-04 14:09:05.002724: I tensorflow/core/platform/profile_utils/cpu_utils.cc:104] CPU Frequency: 2599705000 Hz
2021-05-04 14:09:05.004362: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x5583288d82c0 initialized for platform Host (this does not guarantee that XLA will be used). Devices:
2021-05-04 14:09:05.004381: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version
Traceback (most recent call last):
File "processing/main_processing.py", line 36, in <module>
print(str(server_state.type_signature))
AttributeError: 'list' object has no attribute 'type_signature'
src/participation-hub/federated-learning-microservice/app/slurm-1780.out deleted 100644 → 0
View file @ 1c2b9a52
2021-05-05 12:34:15.886296: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
Traceback (most recent call last):
File "processing/main_processing.py", line 11, in <module>
from federated_training_algorithm import get_federated_algorithm
File "/home/itec/bogdan/Articonf/smart/src/participation-hub/federated-learning-microservice/app/processing/federated_training_algorithm.py", line 48, in <module>
def server_init():
File "/home/itec/bogdan/.conda/envs/MLF/lib/python3.8/site-packages/tensorflow_federated/python/core/impl/wrappers/computation_wrapper.py", line 407, in __call__
result = fn_to_wrap(*args, **kwargs)
File "/home/itec/bogdan/Articonf/smart/src/participation-hub/federated-learning-microservice/app/processing/federated_training_algorithm.py", line 49, in server_init
model = model_fn()
File "/home/itec/bogdan/Articonf/smart/src/participation-hub/federated-learning-microservice/app/processing/models2.py", line 16, in model_fn
return tff.learning.from_keras_model(
NameError: name 'tff' is not defined
src/participation-hub/federated-learning-microservice/app/slurm-1781.out deleted 100644 → 0
View file @ 1c2b9a52
2021-05-05 12:36:34.308397: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
Traceback (most recent call last):
File "processing/main_processing.py", line 11, in <module>
from federated_training_algorithm import get_federated_algorithm
File "/home/itec/bogdan/Articonf/smart/src/participation-hub/federated-learning-microservice/app/processing/federated_training_algorithm.py", line 48, in <module>
def server_init():
File "/home/itec/bogdan/.conda/envs/MLF/lib/python3.8/site-packages/tensorflow_federated/python/core/impl/wrappers/computation_wrapper.py", line 407, in __call__
result = fn_to_wrap(*args, **kwargs)
File "/home/itec/bogdan/Articonf/smart/src/participation-hub/federated-learning-microservice/app/processing/federated_training_algorithm.py", line 49, in server_init
model = model_fn()
File "/home/itec/bogdan/Articonf/smart/src/participation-hub/federated-learning-microservice/app/processing/models2.py", line 19, in model_fn
input_spec=preprocessed_example_dataset.element_spec,
NameError: name 'preprocessed_example_dataset' is not defined
src/participation-hub/federated-learning-microservice/app/slurm-1782.out deleted 100644 → 0
View file @ 1c2b9a52
2021-05-05 12:43:03.812706: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
Traceback (most recent call last):
File "processing/main_processing.py", line 11, in <module>
from federated_training_algorithm import get_federated_algorithm
File "/home/itec/bogdan/Articonf/smart/src/participation-hub/federated-learning-microservice/app/processing/federated_training_algorithm.py", line 48, in <module>
def server_init():
File "/home/itec/bogdan/.conda/envs/MLF/lib/python3.8/site-packages/tensorflow_federated/python/core/impl/wrappers/computation_wrapper.py", line 407, in __call__
result = fn_to_wrap(*args, **kwargs)
File "/home/itec/bogdan/Articonf/smart/src/participation-hub/federated-learning-microservice/app/processing/federated_training_algorithm.py", line 49, in server_init
model = model_fn()
File "/home/itec/bogdan/Articonf/smart/src/participation-hub/federated-learning-microservice/app/processing/models2.py", line 19, in model_fn
input_spec=get_input_spec(),
File "/home/itec/bogdan/Articonf/smart/src/participation-hub/federated-learning-microservice/app/processing/models2.py", line 25, in get_input_spec
return collections.OrderedDict(
NameError: name 'collections' is not defined
src/participation-hub/federated-learning-microservice/app/slurm-1783.out deleted 100644 → 0
View file @ 1c2b9a52
2021-05-05 12:44:48.174832: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
2021-05-05 12:45:01.547785: W tensorflow/stream_executor/platform/default/dso_loader.cc:59] Could not load dynamic library 'libcuda.so.1'; dlerror: libcuda.so.1: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64
2021-05-05 12:45:01.548090: W tensorflow/stream_executor/cuda/cuda_driver.cc:312] failed call to cuInit: UNKNOWN ERROR (303)
2021-05-05 12:45:01.548116: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:156] kernel driver does not appear to be running on this host (mcore2): /proc/driver/nvidia/version does not exist
2021-05-05 12:45:01.624528: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN)to use the following CPU instructions in performance-critical operations: AVX2 FMA
To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
2021-05-05 12:45:01.674208: I tensorflow/core/platform/profile_utils/cpu_utils.cc:104] CPU Frequency: 2599705000 Hz
2021-05-05 12:45:01.676283: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x55727649d8f0 initialized for platform Host (this does not guarantee that XLA will be used). Devices:
2021-05-05 12:45:01.676306: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version
Traceback (most recent call last):
File "processing/main_processing.py", line 56, in <module>
print(merged_state[1])
NameError: name 'merged_state' is not defined
src/participation-hub/federated-learning-microservice/app/slurm-1784.out deleted 100644 → 0
View file @ 1c2b9a52
2021-05-05 12:47:00.299027: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
2021-05-05 12:47:14.153090: W tensorflow/stream_executor/platform/default/dso_loader.cc:59] Could not load dynamic library 'libcuda.so.1'; dlerror: libcuda.so.1: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64
2021-05-05 12:47:14.153503: W tensorflow/stream_executor/cuda/cuda_driver.cc:312] failed call to cuInit: UNKNOWN ERROR (303)
2021-05-05 12:47:14.153524: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:156] kernel driver does not appear to be running on this host (mcore2): /proc/driver/nvidia/version does not exist
2021-05-05 12:47:14.220064: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN)to use the following CPU instructions in performance-critical operations: AVX2 FMA
To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
2021-05-05 12:47:14.250921: I tensorflow/core/platform/profile_utils/cpu_utils.cc:104] CPU Frequency: 2599705000 Hz
2021-05-05 12:47:14.252596: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x55caa388f420 initialized for platform Host (this does not guarantee that XLA will be used). Devices:
2021-05-05 12:47:14.252622: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version
src/participation-hub/federated-learning-microservice/app/slurm-1785.out deleted 100644 → 0
View file @ 1c2b9a52
2021-05-05 12:48:43.541178: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
2021-05-05 12:48:57.548367: W tensorflow/stream_executor/platform/default/dso_loader.cc:59] Could not load dynamic library 'libcuda.so.1'; dlerror: libcuda.so.1: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64
2021-05-05 12:48:57.548773: W tensorflow/stream_executor/cuda/cuda_driver.cc:312] failed call to cuInit: UNKNOWN ERROR (303)
2021-05-05 12:48:57.548823: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:156] kernel driver does not appear to be running on this host (mcore2): /proc/driver/nvidia/version does not exist
2021-05-05 12:48:57.617636: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN)to use the following CPU instructions in performance-critical operations: AVX2 FMA
To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
2021-05-05 12:48:57.658912: I tensorflow/core/platform/profile_utils/cpu_utils.cc:104] CPU Frequency: 2599705000 Hz
2021-05-05 12:48:57.661020: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x55e17f536420 initialized for platform Host (this does not guarantee that XLA will be used). Devices:
2021-05-05 12:48:57.661049: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version
src/participation-hub/federated-learning-microservice/app/slurm-1786.out deleted 100644 → 0
View file @ 1c2b9a52
2021-05-05 12:53:08.876262: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
2021-05-05 12:53:22.787924: W tensorflow/stream_executor/platform/default/dso_loader.cc:59] Could not load dynamic library 'libcuda.so.1'; dlerror: libcuda.so.1: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64
2021-05-05 12:53:22.788250: W tensorflow/stream_executor/cuda/cuda_driver.cc:312] failed call to cuInit: UNKNOWN ERROR (303)
2021-05-05 12:53:22.788293: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:156] kernel driver does not appear to be running on this host (mcore2): /proc/driver/nvidia/version does not exist
Traceback (most recent call last):
File "processing/main_processing.py", line 11, in <module>
from federated_training_algorithm import get_federated_algorithm
File "/home/itec/bogdan/Articonf/smart/src/participation-hub/federated-learning-microservice/app/processing/federated_training_algorithm.py", line 56, in <module>
whimsy_model = MnistModel()
NameError: name 'MnistModel' is not defined
src/participation-hub/federated-learning-microservice/app/slurm-1787.out deleted 100644 → 0
View file @ 1c2b9a52
2021-05-05 12:55:42.203681: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
2021-05-05 12:55:56.142028: W tensorflow/stream_executor/platform/default/dso_loader.cc:59] Could not load dynamic library 'libcuda.so.1'; dlerror: libcuda.so.1: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64
2021-05-05 12:55:56.142493: W tensorflow/stream_executor/cuda/cuda_driver.cc:312] failed call to cuInit: UNKNOWN ERROR (303)
2021-05-05 12:55:56.142516: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:156] kernel driver does not appear to be running on this host (mcore2): /proc/driver/nvidia/version does not exist
2021-05-05 12:55:56.215449: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN)to use the following CPU instructions in performance-critical operations: AVX2 FMA
To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
2021-05-05 12:55:56.225527: I tensorflow/core/platform/profile_utils/cpu_utils.cc:104] CPU Frequency: 2599705000 Hz
2021-05-05 12:55:56.227527: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x55be268bd420 initialized for platform Host (this does not guarantee that XLA will be used). Devices:
2021-05-05 12:55:56.227548: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version
src/participation-hub/federated-learning-microservice/app/slurm-1788.out deleted 100644 → 0
View file @ 1c2b9a52
2021-05-05 13:51:03.378007: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
Traceback (most recent call last):
File "processing/main_processing.py", line 10, in <module>
from models import MnistModel
ModuleNotFoundError: No module named 'models'
src/participation-hub/federated-learning-microservice/app/slurm-1789.out deleted 100644 → 0
View file @ 1c2b9a52
2021-05-05 14:16:11.554269: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
2021-05-05 14:16:28.615094: W tensorflow/stream_executor/platform/default/dso_loader.cc:59] Could not load dynamic library 'libcuda.so.1'; dlerror: libcuda.so.1: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64
2021-05-05 14:16:28.615308: W tensorflow/stream_executor/cuda/cuda_driver.cc:312] failed call to cuInit: UNKNOWN ERROR (303)
2021-05-05 14:16:28.615342: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:156] kernel driver does not appear to be running on this host (mcore2): /proc/driver/nvidia/version does not exist
2021-05-05 14:16:28.696253: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN)to use the following CPU instructions in performance-critical operations: AVX2 FMA
To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
2021-05-05 14:16:28.725734: I tensorflow/core/platform/profile_utils/cpu_utils.cc:104] CPU Frequency: 2599705000 Hz
2021-05-05 14:16:28.727680: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x56288f42e0d0 initialized for platform Host (this does not guarantee that XLA will be used). Devices:
2021-05-05 14:16:28.727704: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version
Traceback (most recent call last):
File "processing/main_processing.py", line 13, in <module>
from evaluation import evaluate
File "/home/itec/bogdan/Articonf/smart/src/participation-hub/federated-learning-microservice/app/processing/evaluation.py", line 4, in <module>
from models import MnistModel
ModuleNotFoundError: No module named 'models'
src/participation-hub/federated-learning-microservice/app/slurm-1790.out deleted 100644 → 0
View file @ 1c2b9a52
2021-05-05 14:19:11.932075: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
2021-05-05 14:19:25.830410: W tensorflow/stream_executor/platform/default/dso_loader.cc:59] Could not load dynamic library 'libcuda.so.1'; dlerror: libcuda.so.1: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64
2021-05-05 14:19:25.830831: W tensorflow/stream_executor/cuda/cuda_driver.cc:312] failed call to cuInit: UNKNOWN ERROR (303)
2021-05-05 14:19:25.830869: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:156] kernel driver does not appear to be running on this host (mcore2): /proc/driver/nvidia/version does not exist
2021-05-05 14:19:25.902017: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN)to use the following CPU instructions in performance-critical operations: AVX2 FMA
To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
2021-05-05 14:19:25.911762: I tensorflow/core/platform/profile_utils/cpu_utils.cc:104] CPU Frequency: 2599705000 Hz
2021-05-05 14:19:25.913382: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x563c6edbf450 initialized for platform Host (this does not guarantee that XLA will be used). Devices:
2021-05-05 14:19:25.913401: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version
src/participation-hub/federated-learning-microservice/app/test.py deleted 100644 → 0
View file @ 1c2b9a52
import collections
import numpy as np
import tensorflow as tf
import tensorflow_federated as tff
#### check if needed vvv
from matplotlib import pyplot as plt
################################
np.random.seed(0)
print(tff.federated_computation(lambda: 'Hello, World!')())
###########################################################
####### LOADING DATASET ###################################
emnist_train, emnist_test = tff.simulation.datasets.emnist.load_data()
example_dataset = emnist_train.create_tf_dataset_for_client(
emnist_train.client_ids[0])
###########################################################
####### Preprocessing data ################################
NUM_CLIENTS = 5
NUM_EPOCHS = 5
BATCH_SIZE = 20
SHUFFLE_BUFFER = 100
PREFETCH_BUFFER = 10
def preprocess(dataset):
def batch_format_fn(element):
"""Flatten a batch `pixels` and return the features as an `OrderedDict`."""
return collections.OrderedDict(
x=tf.reshape(element['pixels'], [-1, 784]),
y=tf.reshape(element['label'], [-1, 1]))
return dataset.repeat(NUM_EPOCHS).shuffle(SHUFFLE_BUFFER).batch(
BATCH_SIZE).map(batch_format_fn).prefetch(PREFETCH_BUFFER)
preprocessed_example_dataset = preprocess(example_dataset)
sample_batch = tf.nest.map_structure(lambda x: x.numpy(),
next(iter(preprocessed_example_dataset)))
print("sample_batch")
print(sample_batch)
def make_federated_data(client_data, client_ids):
return [
preprocess(client_data.create_tf_dataset_for_client(x))
for x in client_ids
]
sample_clients = emnist_train.client_ids[0:NUM_CLIENTS]
federated_train_data = make_federated_data(emnist_train, sample_clients)
print('Number of client datasets: {l}'.format(l=len(federated_train_data)))
print('First dataset: {d}'.format(d=federated_train_data[0]))
#######################################################################################
###### Creating the tutorial model
#######################################################################################
# def create_keras_model():
# return tf.keras.models.Sequential([
# tf.keras.layers.InputLayer(input_shape=(784,)),
# tf.keras.layers.Dense(10, kernel_initializer='zeros'),
# tf.keras.layers.Softmax(),
# ])
# def model_fn():
# # We _must_ create a new model here, and _not_ capture it from an external
# # scope. TFF will call this within different graph contexts.
# keras_model = create_keras_model()
# return tff.learning.from_keras_model(
# keras_model,
# input_spec=preprocessed_example_dataset.element_spec,
# loss=tf.keras.losses.SparseCategoricalCrossentropy(),
# metrics=[tf.keras.metrics.SparseCategoricalAccuracy()])
# #######################################################################################
# ####Tutorial implementation
# iterative_process = tff.learning.build_federated_averaging_process(
# model_fn,
# client_optimizer_fn=lambda: tf.keras.optimizers.SGD(learning_rate=0.02),
# server_optimizer_fn=lambda: tf.keras.optimizers.SGD(learning_rate=1.0))
# print("\nInitialize Iterative Procees signature:")
# print(str(iterative_process.initialize.type_signature)) ## Just a print for info
# state = iterative_process.initialize()
# NUM_ROUNDS = 10
# print("\nStarting {} rounds of training:".format(NUM_ROUNDS))
# for round_num in range(1, NUM_ROUNDS+1):
# state, metrics = iterative_process.next(state, federated_train_data)
# print('round {:2d}, metrics={}'.format(round_num, metrics))
#######################################################################################
####### Customizing the model implementation ##########################################
#######################################################################################
print(" STAAARTIIING")
MnistVariables = collections.namedtuple(
'MnistVariables', 'weights bias num_examples loss_sum accuracy_sum')
def create_mnist_variables():
return MnistVariables(
weights=tf.Variable(
lambda: tf.zeros(dtype=tf.float32, shape=(784, 10)),
name='weights',
trainable=True),
bias=tf.Variable(
lambda: tf.zeros(dtype=tf.float32, shape=(10)),
name='bias',
trainable=True),
num_examples=tf.Variable(0.0, name='num_examples', trainable=False),
loss_sum=tf.Variable(0.0, name='loss_sum', trainable=False),
accuracy_sum=tf.Variable(0.0, name='accuracy_sum', trainable=False))
def mnist_forward_pass(variables, batch):
y = tf.nn.softmax(tf.matmul(batch['x'], variables.weights) + variables.bias)
predictions = tf.cast(tf.argmax(y, 1), tf.int32)
flat_labels = tf.reshape(batch['y'], [-1])
loss = -tf.reduce_mean(
tf.reduce_sum(tf.one_hot(flat_labels, 10) * tf.math.log(y), axis=[1]))
accuracy = tf.reduce_mean(
tf.cast(tf.equal(predictions, flat_labels), tf.float32))
num_examples = tf.cast(tf.size(batch['y']), tf.float32)
variables.num_examples.assign_add(num_examples)
variables.loss_sum.assign_add(loss * num_examples)
variables.accuracy_sum.assign_add(accuracy * num_examples)
return loss, predictions
def get_local_mnist_metrics(variables):
return collections.OrderedDict(
num_examples=variables.num_examples,
loss=variables.loss_sum / variables.num_examples,
accuracy=variables.accuracy_sum / variables.num_examples)
@tff.federated_computation
def aggregate_mnist_metrics_across_clients(metrics):
return collections.OrderedDict(
num_examples=tff.federated_sum(metrics.num_examples),
loss=tff.federated_mean(metrics.loss, metrics.num_examples),
accuracy=tff.federated_mean(metrics.accuracy, metrics.num_examples))
class MnistModel(tff.learning.Model):
def __init__(self):
self._variables = create_mnist_variables()
@property
def trainable_variables(self):
return [self._variables.weights, self._variables.bias]
@property
def non_trainable_variables(self):
return []
@property
def local_variables(self):
return [
self._variables.num_examples, self._variables.loss_sum,
self._variables.accuracy_sum
]
@property
def input_spec(self):
return collections.OrderedDict(
x=tf.TensorSpec([None, 784], tf.float32),
y=tf.TensorSpec([None, 1], tf.int32))
@tf.function
def forward_pass(self, batch, training=True):
del training
loss, predictions = mnist_forward_pass(self._variables, batch)
num_exmaples = tf.shape(batch['x'])[0]
return tff.learning.BatchOutput(
loss=loss, predictions=predictions, num_examples=num_exmaples)
@tf.function
def report_local_outputs(self):
return get_local_mnist_metrics(self._variables)
@property
def federated_output_computation(self):
return aggregate_mnist_metrics_across_clients
iterative_process = tff.learning.build_federated_averaging_process(
MnistModel,
client_optimizer_fn=lambda: tf.keras.optimizers.SGD(learning_rate=0.02))
state = iterative_process.initialize()
state, metrics = iterative_process.next(state, federated_train_data)
NUM_ROUNDS = 10
print("\nStarting {} rounds of training:".format(NUM_ROUNDS))
for round_num in range(1, NUM_ROUNDS + 1):
state, metrics = iterative_process.next(state, federated_train_data)
print('round {:2d}, metrics={}'.format(round_num, metrics))
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment