Functional prototype fake news semi-fed model

src/participation-hub/federated-learning-microservice/app/mylog

@@ -50,3 +50,15 @@ Submitted batch job 1769
 Submitted batch job 1777
 Submitted batch job 1778
 Submitted batch job 1779
+Submitted batch job 1780
+Submitted batch job 1781
+Submitted batch job 1782
+Submitted batch job 1783
+Submitted batch job 1784
+Submitted batch job 1785
+Submitted batch job 1786
+Submitted batch job 1787
+Submitted batch job 1788
+Submitted batch job 1789
+Submitted batch job 1790
+Submitted batch job 1791

src/participation-hub/federated-learning-microservice/app/processing/evaluation.py → src/participation-hub/federated-learning-microservice/app/processing/fake_news/evaluation.py

 import numpy as np
 import tensorflow as tf
 import tensorflow_federated as tff
-from models import MnistModel
+#from models import MnistModel
+from models2 import model_fn
 
 
 def evaluate(server_state):
-  keras_model = MnistModel()
+  keras_model = model_fn
   keras_model.compile(
       loss=tf.keras.losses.SparseCategoricalCrossentropy(),
       metrics=[tf.keras.metrics.SparseCategoricalAccuracy()]  

src/participation-hub/federated-learning-microservice/app/processing/federated_training_algorithm.py → src/participation-hub/federated-learning-microservice/app/processing/fake_news/federated_training_algorithm copy.py

@@ -2,6 +2,7 @@ import numpy as np
 import tensorflow as tf
 import tensorflow_federated as tff
 from models import MnistModel
+from models2 import model_fn
 
 #tf_dataset_type = None
 #model_weights_type = None

src/participation-hub/federated-learning-microservice/app/processing/fake_news/federated_training_algorithm.py

+import numpy as np
+import tensorflow as tf
+import tensorflow_federated as tff
+#from models import MnistModel
+from models2 import model_fn
+
+#tf_dataset_type = None
+#model_weights_type = None
+
+@tf.function
+def client_update(model, dataset, server_weights, client_optimizer):
+  """Performs training (using the server model weights) on the client's dataset."""
+  # Initialize the client model with the current server weights.
+  client_weights = model.trainable_variables
+  # Assign the server weights to the client model.
+  tf.nest.map_structure(lambda x, y: x.assign(y),
+                        client_weights, server_weights)
+
+  # Use the client_optimizer to update the local model.
+  for batch in dataset:
+    with tf.GradientTape() as tape:
+      # Compute a forward pass on the batch of data
+      outputs = model.forward_pass(batch)
+
+    # Compute the corresponding gradient
+    grads = tape.gradient(outputs.loss, client_weights)
+    grads_and_vars = zip(grads, client_weights)
+
+    # Apply the gradient using a client optimizer.
+    client_optimizer.apply_gradients(grads_and_vars)
+
+  return client_weights
+
+@tf.function
+def server_update(model, mean_client_weights):
+  """Updates the server model weights as the average of the client model weights."""
+  model_weights = model.trainable_variables
+  # Assign the mean client weights to the server model.
+  tf.nest.map_structure(lambda x, y: x.assign(y),
+                        model_weights, mean_client_weights)
+  return model_weights
+
+
+
+
+#Creating the initialization computation
+@tff.tf_computation
+def server_init():
+  model = model_fn()
+  return model.trainable_variables
+
+@tff.federated_computation
+def initialize_fn():
+  return tff.federated_value(server_init(), tff.SERVER)
+
+whimsy_model = model_fn()
+tf_dataset_type = tff.SequenceType(whimsy_model.input_spec)
+model_weights_type = server_init.type_signature.result
+
+@tff.tf_computation(tf_dataset_type, model_weights_type)
+def client_update_fn(tf_dataset, server_weights):
+  model = model_fn()
+  client_optimizer = tf.keras.optimizers.SGD(learning_rate=0.01)
+  return client_update(model, tf_dataset, server_weights, client_optimizer)
+
+@tff.tf_computation(model_weights_type)
+def server_update_fn(mean_client_weights):
+  model = model_fn()
+  return server_update(model, mean_client_weights)
+
+federated_server_type = tff.FederatedType(model_weights_type, tff.SERVER)
+federated_dataset_type = tff.FederatedType(tf_dataset_type, tff.CLIENTS)
+
+@tff.federated_computation(federated_server_type, federated_dataset_type)
+def next_fn(server_weights, federated_dataset):
+  # Broadcast the server weights to the clients.
+  print("server_weights")
+  print(str(server_weights.type_signature))
+  server_weights_at_client = tff.federated_broadcast(server_weights)
+
+  # Each client computes their updated weights.
+  client_weights = tff.federated_map(
+      client_update_fn, (federated_dataset, server_weights_at_client))
+
+  # The server averages these updates.
+  mean_client_weights = tff.federated_mean(client_weights)
+
+  print("mean_client_wieghts")
+  print(str(mean_client_weights.type_signature))
+
+  # The server updates its model.
+  server_weights = tff.federated_map(server_update_fn, mean_client_weights)
+
+  return server_weights
+
+def get_federated_algorithm():
+    #Creating the next_fn
+
+    #Getting the data type, needed explicitily in the modell functions
+    whimsy_model = model_fn()
+    global tf_dataset_type
+    tf_dataset_type = tff.SequenceType(whimsy_model.input_spec)
+    print("tf_dataset_type")
+    print(str(tf_dataset_type))
+    global model_weights_type
+    model_weights_type = server_init.type_signature.result
+    print("model_weights_type")
+    print(str(model_weights_type))
+    # finished printing types
+
+    federated_server_type = tff.FederatedType(model_weights_type, tff.SERVER)
+    federated_dataset_type = tff.FederatedType(tf_dataset_type, tff.CLIENTS)
+
+    federated_algorithm = tff.templates.IterativeProcess(
+        initialize_fn=initialize_fn,
+        next_fn=next_fn
+    )
+    return federated_algorithm
+
+
+
+def merge_2_states(server1_state, server2_state):
+  return np.mean( np.array([ server1_state, server2_state ]), axis=0 )
\ No newline at end of file

src/participation-hub/federated-learning-microservice/app/processing/main_processing.py → src/participation-hub/federated-learning-microservice/app/processing/fake_news/main_processing.py

@@ -7,7 +7,7 @@ import tensorflow_federated as tff
 from preprocessing import get_client_ids
 from preprocessing import get_federated_train_data
 from preprocessing import preprocess
-from models import MnistModel
+#from models import MnistModel
 from federated_training_algorithm import get_federated_algorithm
 from federated_training_algorithm import merge_2_states
 from evaluation import evaluate
@@ -44,7 +44,7 @@ for round in range(2):
   server2_state = federated_algorithm.next(server2_state, federated_train_data)
 
 
-
+merged_state = merge_2_states(server_state,server2_state)
 
 print("server_state[1]")
 print(server_state[1])

src/participation-hub/federated-learning-microservice/app/processing/fake_news/models2.py

+import numpy as np
+import tensorflow as tf
+import tensorflow_federated as tff
+import collections
+
+def create_keras_model(): #### DEFAULT TEST MODEL
+  return tf.keras.models.Sequential([
+      tf.keras.layers.InputLayer(input_shape=(784,)),
+      tf.keras.layers.Dense(10, kernel_initializer='zeros'),
+      tf.keras.layers.Softmax(),
+  ])
+
+
+def model_fn():
+  # We _must_ create a new model here, and _not_ capture it from an external
+  # scope. TFF will call this within different graph contexts.
+  keras_model = create_keras_model() ###TODO CAN BE CHANGED TO INCLUDE CALLS TO OTHER MODELS (or it cannot, because you cannot specify which model you want to call?)
+  return tff.learning.from_keras_model(
+      keras_model,
+      input_spec=get_input_spec(),
+      loss=tf.keras.losses.SparseCategoricalCrossentropy(),
+      metrics=[tf.keras.metrics.SparseCategoricalAccuracy()])
+
+
+def get_input_spec():
+    return collections.OrderedDict(
+        x=tf.TensorSpec([None, 784], tf.float32),
+        y=tf.TensorSpec([None, 1], tf.int32))
\ No newline at end of file

src/participation-hub/federated-learning-microservice/app/processing/fake_news/preprocessing copy.py

+import collections
+import attr
+import functools
+import numpy as np
+import tensorflow as tf
+import tensorflow_federated as tff
+
+np.random.seed(0)
+
+emnist_train, emnist_test = tff.simulation.datasets.emnist.load_data()
+
+# NUM_CLIENTS = 10
+# BATCH_SIZE = 20
+
+def preprocess(dataset, BATCH_SIZE = 20):
+
+  def batch_format_fn(element):
+    """Flatten a batch of EMNIST data and return a (features, label) tuple."""
+    return (tf.reshape(element['pixels'], [-1, 784]), 
+            tf.reshape(element['label'], [-1, 1]))
+
+  return dataset.batch(BATCH_SIZE).map(batch_format_fn)
+
+def get_client_ids(emnist_train, NUM_CLIENTS = 10):
+    return np.random.choice(emnist_train.client_ids, size=NUM_CLIENTS, replace=False)
+    
+
+#client_ids = np.random.choice(emnist_train.client_ids, size=NUM_CLIENTS, replace=False)
+
+def get_federated_train_data(emnist_train, NUM_CLIENTS = 10):
+    client_ids = get_client_ids(emnist_train, NUM_CLIENTS)
+    return [preprocess(emnist_train.create_tf_dataset_for_client(x))
+                for x in client_ids
+                ]
+
+
+
+# federated_train_data = [preprocess(emnist_train.create_tf_dataset_for_client(x))
+#   for x in client_ids
+# ]
+
+##################################################################
+## Second Dataset
+
+############################################
+def preprocess_and_shuffle(dataset):
+  """Applies `preprocess_dataset` above and shuffles the result."""
+  preprocessed = preprocess(dataset)
+  return preprocessed.shuffle(buffer_size=5)
+
+
+
+
+

src/participation-hub/federated-learning-microservice/app/processing/fake_news/preprocessing.py

+import collections
+import attr
+import functools
+import numpy as np
+import tensorflow as tf
+import tensorflow_federated as tff
+
+np.random.seed(0)
+
+emnist_train, emnist_test = tff.simulation.datasets.emnist.load_data()
+
+# NUM_CLIENTS = 10
+# BATCH_SIZE = 20
+
+def preprocess(dataset, BATCH_SIZE = 20):
+
+  def batch_format_fn(element):
+    """Flatten a batch of EMNIST data and return a (features, label) tuple."""
+    return (tf.reshape(element['pixels'], [-1, 784]), 
+            tf.reshape(element['label'], [-1, 1]))
+
+  return dataset.batch(BATCH_SIZE).map(batch_format_fn)
+
+def get_client_ids(emnist_train, NUM_CLIENTS = 10):
+    return np.random.choice(emnist_train.client_ids, size=NUM_CLIENTS, replace=False)
+    
+
+#client_ids = np.random.choice(emnist_train.client_ids, size=NUM_CLIENTS, replace=False)
+
+def get_federated_train_data(emnist_train, NUM_CLIENTS = 10):
+    client_ids = get_client_ids(emnist_train, NUM_CLIENTS)
+    return [preprocess(emnist_train.create_tf_dataset_for_client(x))
+                for x in client_ids
+                ]
+
+######
+def preprocess_and_shuffle(dataset):
+  """Applies `preprocess_dataset` above and shuffles the result."""
+  preprocessed = preprocess(dataset)
+  return preprocessed.shuffle(buffer_size=5)
+
+
+
+#########################################################################################
+#########################################################################################
+#########################################################################################

src/participation-hub/federated-learning-microservice/app/processing/fake_news/prototype_federat_data copy.py

+import pandas as pd
+import re
+import matplotlib.pyplot as plt
+from sklearn.model_selection import train_test_split
+from sklearn.preprocessing import LabelEncoder
+from tensorflow.keras.models import Model
+from tensorflow.keras.models import Sequential
+from tensorflow.keras.layers import LSTM, Activation, Dense, Dropout, Input, Embedding
+from tensorflow.keras.optimizers import RMSprop
+from tensorflow.keras.preprocessing.text import Tokenizer
+from tensorflow.keras.preprocessing import sequence
+from tensorflow.keras.callbacks import EarlyStopping
+from tensorflow.keras.preprocessing.sequence import pad_sequences
+
+real = pd.read_csv("processing/fake_news/prototype_db_fake_real/True.csv")
+fake = pd.read_csv("processing/fake_news/prototype_db_fake_real/Fake.csv")
+
+# dropping rows that have urls as text and date, real's dates look fine, also dropping ones that have no text
+fake_drop = fake.drop(index=[9358,15507,15508,18933])
+fake_drop = fake_drop.drop(fake_drop.loc[fake_drop.text == ' '].index)
+real_drop = real.drop(real.loc[real.text == ' '].index)
+
+# Give labels to data before combining
+fake['fake'] = 1
+real['fake'] = 0
+combined = pd.concat([fake, real])
+
+no_reuters = combined.copy()
+no_reuters.text = no_reuters.text.str.replace('Reuters', '')
+combined = no_reuters.copy()
+## train/test split the text data and labels
+features = combined['text']
+labels = combined['fake']
+X_train, X_test, y_train, y_test = train_test_split(features, labels, random_state = 42)
+
+# the model will remember only the top 2000 most common words
+max_words = 2000
+max_len = 400
+
+token = Tokenizer(num_words=max_words, lower=True, split=' ')
+token.fit_on_texts(X_train.values)
+sequences = token.texts_to_sequences(X_train.values)
+train_sequences_padded = pad_sequences(sequences, maxlen=max_len)
+
+embed_dim = 50
+lstm_out = 64
+batch_size = 32
+input_shape_var = (max_words, )
+model = Sequential()
+model.add(Embedding(max_words, embed_dim, input_length = max_len, input_shape=input_shape_var))
+model.add(LSTM(lstm_out))
+model.add(Dense(256))
+model.add(Activation('relu'))
+model.add(Dropout(0.5))
+model.add(Dense(1, name='out_layer'))
+model.add(Activation('sigmoid'))
+model.compile(loss = 'binary_crossentropy', optimizer='adam',\
+               metrics = ['accuracy'])
+print(model.summary())
+
+print()
+
+history = model.fit(train_sequences_padded, y_train, batch_size=batch_size, epochs = 5, validation_split=0.2)
+test_sequences = token.texts_to_sequences(X_test)
+test_sequences_padded = pad_sequences(test_sequences, maxlen=max_len)
+model.evaluate(test_sequences_padded, y_test)
+
+
+###################################################################
+
+experiment_name = "mnist"
+method = "tff_training"
+client_lr = 1e-2
+server_lr = 1e-2
+split = 4
+NUM_ROUNDS = 5
+NUM_EPOCHS = 5
+BATCH_SIZE = 20
+PREFETCH_BUFFER = 10
+
+this_dir = Path.cwd()
+model_dir = this_dir / "saved_models" / experiment_name / method
+output_dir = this_dir / "results" / experiment_name / method
+
+if not model_dir.exists():
+    model_dir.mkdir(parents=True)
+
+if not output_dir.exists():
+    output_dir.mkdir(parents=True)
+
+(x_train, y_train), (x_test, y_test) = mnist.load_data()
+
+x_train = x_train.astype(np.float32)
+y_train = y_train.astype(np.int32)
+x_test = x_test.astype(np.float32).reshape(10000, 28, 28, 1)
+y_test = y_test.astype(np.int32).reshape(10000, 1)
+
+total_image_count = len(x_train)
+image_per_set = int(np.floor(total_image_count/split))
+
+client_train_dataset = collections.OrderedDict()
+for i in range(1, split+1):
+    client_name = "client_" + str(i)
+    start = image_per_set * (i-1)
+    end = image_per_set * i
+
+    print(f"Adding data from {start} to {end} for client : {client_name}")
+    data = collections.OrderedDict((('label', y_train[start:end]), ('pixels', x_train[start:end])))
+    client_train_dataset[client_name] = data
+
+train_dataset = tff.simulation.FromTensorSlicesClientData(client_train_dataset)
+
+sample_dataset = train_dataset.create_tf_dataset_for_client(train_dataset.client_ids[0])
+sample_element = next(iter(sample_dataset))
+
+SHUFFLE_BUFFER = image_per_set
\ No newline at end of file

src/participation-hub/federated-learning-microservice/app/processing/fake_news/prototype_fkns.py

+
+# import numpy as np ##
+# import pandas as pd ##
+
+# import seaborn as sns
+# import matplotlib.pyplot as plt
+# import nltk ##
+# from sklearn.preprocessing import LabelBinarizer
+# from nltk.corpus import stopwords ##
+# from nltk.stem.porter import PorterStemmer #
+# from wordcloud import WordCloud,STOPWORDS ##
+# from nltk.stem import WordNetLemmatizer #
+# from nltk.tokenize import word_tokenize,sent_tokenize #
+# from bs4 import BeautifulSoup ##
+# import re,string,unicodedata ##
+# from keras.preprocessing import text, sequence
+# from sklearn.metrics import classification_report,confusion_matrix,accuracy_score
+# from sklearn.model_selection import train_test_split
+# from string import punctuation ##
+# from nltk import pos_tag #
+# from nltk.corpus import wordnet #
+# import keras
+# from keras.models import Sequential
+# from keras.layers import Dense,Embedding,LSTM,Dropout
+# from keras.callbacks import ReduceLROnPlateau
+# import tensorflow as tf
+# import tensorflow_federated as tff
+
+import pandas as pd
+import re
+import matplotlib.pyplot as plt
+from sklearn.model_selection import train_test_split
+from sklearn.preprocessing import LabelEncoder
+from tensorflow.keras.models import Model
+from tensorflow.keras.models import Sequential
+from tensorflow.keras.layers import LSTM, Activation, Dense, Dropout, Input, Embedding
+from tensorflow.keras.optimizers import RMSprop
+from tensorflow.keras.preprocessing.text import Tokenizer
+from tensorflow.keras.preprocessing import sequence
+from tensorflow.keras.callbacks import EarlyStopping
+from tensorflow.keras.preprocessing.sequence import pad_sequences
+
+
+# def get_dataframe_OLD():
+#     true = pd.read_csv("processing/fake_news/prototype_db_fake_real/True.csv")
+#     false = pd.read_csv("processing/fake_news/prototype_db_fake_real/Fake.csv")
+#     true['category'] = 1
+#     false['category'] = 0
+#     df = pd.concat([true,false]) #Merging the 2 datasets
+#     df['text'] = df['text'] + " " + df['title']
+#     del df['title']
+#     del df['subject']
+#     del df['date']
+
+#     return df
+
+# def strip_html(text):
+#     soup = BeautifulSoup(text, "html.parser")
+#     return soup.get_text()
+
+# #Removing the square brackets
+# def remove_between_square_brackets(text):
+#     return re.sub('\[[^]]*\]', '', text)
+# # Removing URL's
+# def remove_between_square_brackets(text):
+#     return re.sub(r'http\S+', '', text)
+# #Removing the stopwords from text
+# def remove_stopwords(text):
+#     stop = set(nltk.corpus.stopwords.words('english'))
+#     punctuation = list(string.punctuation)
+#     stop.update(punctuation)
+
+#     final_text = []
+#     for i in text.split():
+#         if i.strip().lower() not in stop:
+#             final_text.append(i.strip())
+#     return " ".join(final_text)
+# #Removing the noisy text
+# def denoise_text(text):
+#     text = strip_html(text)
+#     text = remove_between_square_brackets(text)
+#     text = remove_stopwords(text)
+#     return text
+# #Apply function on review column
+
+
+# def data_cleaning():   
+#     df = get_dataframe()
+#     df['text']=df['text'].apply(denoise_text)
+
+#     print("#####")
+
+#data_cleaning()
+    
+# x_train,x_test,y_train,y_test = train_test_split(df.text,df.category,random_state = 0)
+
+# max_features = 10000
+# maxlen = 300
+
+# tokenizer = text.Tokenizer(num_words=max_features)
+# tokenizer.fit_on_texts(x_train)
+# tokenized_train = tokenizer.texts_to_sequences(x_train)
+# x_train = sequence.pad_sequences(tokenized_train, maxlen=maxlen)
+
+# #Defining Neural Network
+# model = Sequential()
+# #Non-trainable embeddidng layer
+# model.add(Embedding(max_features, output_dim=embed_size, weights=[embedding_matrix], input_length=maxlen, trainable=False))
+# #LSTM 
+# model.add(LSTM(units=128 , return_sequences = True , recurrent_dropout = 0.25 , dropout = 0.25))
+# model.add(LSTM(units=64 , recurrent_dropout = 0.1 , dropout = 0.1))
+# model.add(Dense(units = 32 , activation = 'relu'))
+# model.add(Dense(1, activation='sigmoid'))
+# model.compile(optimizer=keras.optimizers.Adam(lr = 0.01), loss='binary_crossentropy', metrics=['accuracy'])
+
+
+
+real = pd.read_csv("processing/fake_news/prototype_db_fake_real/True.csv")
+fake = pd.read_csv("processing/fake_news/prototype_db_fake_real/Fake.csv")
+
+# dropping rows that have urls as text and date, real's dates look fine, also dropping ones that have no text
+fake_drop = fake.drop(index=[9358,15507,15508,18933])
+fake_drop = fake_drop.drop(fake_drop.loc[fake_drop.text == ' '].index)
+real_drop = real.drop(real.loc[real.text == ' '].index)
+
+# Give labels to data before combining
+fake['fake'] = 1
+real['fake'] = 0
+combined = pd.concat([fake, real])
+
+no_reuters = combined.copy()
+no_reuters.text = no_reuters.text.str.replace('Reuters', '')
+combined = no_reuters.copy()
+## train/test split the text data and labels
+features = combined['text']
+labels = combined['fake']
+X_train, X_test, y_train, y_test = train_test_split(features, labels, random_state = 42)
+
+# the model will remember only the top 2000 most common words
+max_words = 2000
+max_len = 400
+
+token = Tokenizer(num_words=max_words, lower=True, split=' ')
+token.fit_on_texts(X_train.values)
+sequences = token.texts_to_sequences(X_train.values)
+train_sequences_padded = pad_sequences(sequences, maxlen=max_len)
+
+embed_dim = 50
+lstm_out = 64
+batch_size = 32
+input_shape_var = (max_words, )
+model = Sequential()
+model.add(Embedding(max_words, embed_dim, input_length = max_len, input_shape=input_shape_var))
+model.add(LSTM(lstm_out))
+model.add(Dense(256))
+model.add(Activation('relu'))
+model.add(Dropout(0.5))
+model.add(Dense(1, name='out_layer'))
+model.add(Activation('sigmoid'))
+model.compile(loss = 'binary_crossentropy', optimizer='adam',\
+               metrics = ['accuracy'])
+print(model.summary())
+
+print()
+
+history = model.fit(train_sequences_padded, y_train, batch_size=batch_size, epochs = 5, validation_split=0.2)
+test_sequences = token.texts_to_sequences(X_test)
+test_sequences_padded = pad_sequences(test_sequences, maxlen=max_len)
+model.evaluate(test_sequences_padded, y_test)
\ No newline at end of file

src/participation-hub/federated-learning-microservice/app/processing/prototype/evaluation.py

+import numpy as np
+import tensorflow as tf
+import tensorflow_federated as tff
+#from models import MnistModel
+from models2 import model_fn
+
+
+def evaluate(server_state):
+  keras_model = model_fn
+  keras_model.compile(
+      loss=tf.keras.losses.SparseCategoricalCrossentropy(),
+      metrics=[tf.keras.metrics.SparseCategoricalAccuracy()]  
+  )
+  keras_model.set_weights(server_state)
+  keras_model.evaluate(central_emnist_test)
+
+
+def evaluate2(server_state,tff_learning_model):
+  
+  #TODO: Assign weights to the model
+  #First idea = server_update function??
+
+  evaluation = tff.learning.build_federated_evaluation(tff_learning_model)
+  
+  
+  # keras_model = MnistModel()
+  # keras_model.compile(
+  #     loss=tf.keras.losses.SparseCategoricalCrossentropy(),
+  #     metrics=[tf.keras.metrics.SparseCategoricalAccuracy()]  
+  # )
+  # keras_model.set_weights(server_state)
+  # keras_model.evaluate(central_emnist_test)

src/participation-hub/federated-learning-microservice/app/processing/prototype/federated_training_algorithm copy.py

+import numpy as np
+import tensorflow as tf
+import tensorflow_federated as tff
+from models import MnistModel
+from models2 import model_fn
+
+#tf_dataset_type = None
+#model_weights_type = None
+
+@tf.function
+def client_update(model, dataset, server_weights, client_optimizer):
+  """Performs training (using the server model weights) on the client's dataset."""
+  # Initialize the client model with the current server weights.
+  client_weights = model.trainable_variables
+  # Assign the server weights to the client model.
+  tf.nest.map_structure(lambda x, y: x.assign(y),
+                        client_weights, server_weights)
+
+  # Use the client_optimizer to update the local model.
+  for batch in dataset:
+    with tf.GradientTape() as tape:
+      # Compute a forward pass on the batch of data
+      outputs = model.forward_pass(batch)
+
+    # Compute the corresponding gradient
+    grads = tape.gradient(outputs.loss, client_weights)
+    grads_and_vars = zip(grads, client_weights)
+
+    # Apply the gradient using a client optimizer.
+    client_optimizer.apply_gradients(grads_and_vars)
+
+  return client_weights
+
+@tf.function
+def server_update(model, mean_client_weights):
+  """Updates the server model weights as the average of the client model weights."""
+  model_weights = model.trainable_variables
+  # Assign the mean client weights to the server model.
+  tf.nest.map_structure(lambda x, y: x.assign(y),
+                        model_weights, mean_client_weights)
+  return model_weights
+
+
+
+
+#Creating the initialization computation
+@tff.tf_computation
+def server_init():
+  model = MnistModel() #model_fn()
+  return model.trainable_variables
+
+@tff.federated_computation
+def initialize_fn():
+  return tff.federated_value(server_init(), tff.SERVER)
+
+whimsy_model = MnistModel()
+tf_dataset_type = tff.SequenceType(whimsy_model.input_spec)
+model_weights_type = server_init.type_signature.result
+
+@tff.tf_computation(tf_dataset_type, model_weights_type)
+def client_update_fn(tf_dataset, server_weights):
+  model = MnistModel()#model_fn()
+  client_optimizer = tf.keras.optimizers.SGD(learning_rate=0.01)
+  return client_update(model, tf_dataset, server_weights, client_optimizer)
+
+@tff.tf_computation(model_weights_type)
+def server_update_fn(mean_client_weights):
+  model = MnistModel()#model_fn()
+  return server_update(model, mean_client_weights)
+
+federated_server_type = tff.FederatedType(model_weights_type, tff.SERVER)
+federated_dataset_type = tff.FederatedType(tf_dataset_type, tff.CLIENTS)
+
+@tff.federated_computation(federated_server_type, federated_dataset_type)
+def next_fn(server_weights, federated_dataset):
+  # Broadcast the server weights to the clients.
+  print("server_weights")
+  print(str(server_weights.type_signature))
+  server_weights_at_client = tff.federated_broadcast(server_weights)
+
+  # Each client computes their updated weights.
+  client_weights = tff.federated_map(
+      client_update_fn, (federated_dataset, server_weights_at_client))
+
+  # The server averages these updates.
+  mean_client_weights = tff.federated_mean(client_weights)
+
+  print("mean_client_wieghts")
+  print(str(mean_client_weights.type_signature))
+
+  # The server updates its model.
+  server_weights = tff.federated_map(server_update_fn, mean_client_weights)
+
+  return server_weights
+
+def get_federated_algorithm():
+    #Creating the next_fn
+
+    #Getting the data type, needed explicitily in the modell functions
+    whimsy_model = MnistModel() #model_fn()
+    global tf_dataset_type
+    tf_dataset_type = tff.SequenceType(whimsy_model.input_spec)
+    print("tf_dataset_type")
+    print(str(tf_dataset_type))
+    global model_weights_type
+    model_weights_type = server_init.type_signature.result
+    print("model_weights_type")
+    print(str(model_weights_type))
+    # finished printing types
+
+    federated_server_type = tff.FederatedType(model_weights_type, tff.SERVER)
+    federated_dataset_type = tff.FederatedType(tf_dataset_type, tff.CLIENTS)
+
+    federated_algorithm = tff.templates.IterativeProcess(
+        initialize_fn=initialize_fn,
+        next_fn=next_fn
+    )
+    return federated_algorithm
+
+
+
+def merge_2_states(server_state1, server_state2):
+  return np.mean( np.array([ server_state, server2_state ]), axis=0 )
\ No newline at end of file

src/participation-hub/federated-learning-microservice/app/processing/prototype/federated_training_algorithm.py

+import numpy as np
+import tensorflow as tf
+import tensorflow_federated as tff
+#from models import MnistModel
+from models2 import model_fn
+
+#tf_dataset_type = None
+#model_weights_type = None
+
+@tf.function
+def client_update(model, dataset, server_weights, client_optimizer):
+  """Performs training (using the server model weights) on the client's dataset."""
+  # Initialize the client model with the current server weights.
+  client_weights = model.trainable_variables
+  # Assign the server weights to the client model.
+  tf.nest.map_structure(lambda x, y: x.assign(y),
+                        client_weights, server_weights)
+
+  # Use the client_optimizer to update the local model.
+  for batch in dataset:
+    with tf.GradientTape() as tape:
+      # Compute a forward pass on the batch of data
+      outputs = model.forward_pass(batch)
+
+    # Compute the corresponding gradient
+    grads = tape.gradient(outputs.loss, client_weights)
+    grads_and_vars = zip(grads, client_weights)
+
+    # Apply the gradient using a client optimizer.
+    client_optimizer.apply_gradients(grads_and_vars)
+
+  return client_weights
+
+@tf.function
+def server_update(model, mean_client_weights):
+  """Updates the server model weights as the average of the client model weights."""
+  model_weights = model.trainable_variables
+  # Assign the mean client weights to the server model.
+  tf.nest.map_structure(lambda x, y: x.assign(y),
+                        model_weights, mean_client_weights)
+  return model_weights
+
+
+
+
+#Creating the initialization computation
+@tff.tf_computation
+def server_init():
+  model = model_fn()
+  return model.trainable_variables
+
+@tff.federated_computation
+def initialize_fn():
+  return tff.federated_value(server_init(), tff.SERVER)
+
+whimsy_model = model_fn()
+tf_dataset_type = tff.SequenceType(whimsy_model.input_spec)
+model_weights_type = server_init.type_signature.result
+
+@tff.tf_computation(tf_dataset_type, model_weights_type)
+def client_update_fn(tf_dataset, server_weights):
+  model = model_fn()
+  client_optimizer = tf.keras.optimizers.SGD(learning_rate=0.01)
+  return client_update(model, tf_dataset, server_weights, client_optimizer)
+
+@tff.tf_computation(model_weights_type)
+def server_update_fn(mean_client_weights):
+  model = model_fn()
+  return server_update(model, mean_client_weights)
+
+federated_server_type = tff.FederatedType(model_weights_type, tff.SERVER)
+federated_dataset_type = tff.FederatedType(tf_dataset_type, tff.CLIENTS)
+
+@tff.federated_computation(federated_server_type, federated_dataset_type)
+def next_fn(server_weights, federated_dataset):
+  # Broadcast the server weights to the clients.
+  print("server_weights")
+  print(str(server_weights.type_signature))
+  server_weights_at_client = tff.federated_broadcast(server_weights)
+
+  # Each client computes their updated weights.
+  client_weights = tff.federated_map(
+      client_update_fn, (federated_dataset, server_weights_at_client))
+
+  # The server averages these updates.
+  mean_client_weights = tff.federated_mean(client_weights)
+
+  print("mean_client_wieghts")
+  print(str(mean_client_weights.type_signature))
+
+  # The server updates its model.
+  server_weights = tff.federated_map(server_update_fn, mean_client_weights)
+
+  return server_weights
+
+def get_federated_algorithm():
+    #Creating the next_fn
+
+    #Getting the data type, needed explicitily in the modell functions
+    whimsy_model = model_fn()
+    global tf_dataset_type
+    tf_dataset_type = tff.SequenceType(whimsy_model.input_spec)
+    print("tf_dataset_type")
+    print(str(tf_dataset_type))
+    global model_weights_type
+    model_weights_type = server_init.type_signature.result
+    print("model_weights_type")
+    print(str(model_weights_type))
+    # finished printing types
+
+    federated_server_type = tff.FederatedType(model_weights_type, tff.SERVER)
+    federated_dataset_type = tff.FederatedType(tf_dataset_type, tff.CLIENTS)
+
+    federated_algorithm = tff.templates.IterativeProcess(
+        initialize_fn=initialize_fn,
+        next_fn=next_fn
+    )
+    return federated_algorithm
+
+
+
+def merge_2_states(server1_state, server2_state):
+  return np.mean( np.array([ server1_state, server2_state ]), axis=0 )
\ No newline at end of file

src/participation-hub/federated-learning-microservice/app/processing/prototype/main_processing.py

+import collections
+import attr
+import functools
+import numpy as np
+import tensorflow as tf
+import tensorflow_federated as tff
+from preprocessing import get_client_ids
+from preprocessing import get_federated_train_data
+from preprocessing import preprocess
+#from models import MnistModel
+from federated_training_algorithm import get_federated_algorithm
+from federated_training_algorithm import merge_2_states
+from evaluation import evaluate
+
+np.random.seed(0)
+print("## Starting...")
+
+#GET THE DATA (for now it's the default dataset)
+emnist_train, emnist_test = tff.simulation.datasets.emnist.load_data()
+
+######client_ids = get_client_ids(emnist_train) not used
+print("## Preprocessing the federated_train_data")
+federated_train_data = get_federated_train_data(emnist_train)
+
+print("## Declaring the model")
+#it is done in models.py
+print("## Declaring the federated algorithm")
+
+federated_algorithm = get_federated_algorithm()
+
+server_state = federated_algorithm.initialize()
+
+for round in range(20):
+  server_state = federated_algorithm.next(server_state, federated_train_data)
+
+print("server_state type")
+print(str(type(server_state)))
+print(str(type(server_state[0])))
+
+print("FINISHEEED")
+
+server2_state = federated_algorithm.initialize()
+for round in range(2):
+  server2_state = federated_algorithm.next(server2_state, federated_train_data)
+
+
+merged_state = merge_2_states(server_state,server2_state)
+
+print("server_state[1]")
+print(server_state[1])
+
+print("server2_state[1]")
+print(server2_state[1])
+
+print("merged_state[1]")
+print(merged_state[1])
+
+
+
+
+# print("federated_algorithm.initialize.type_signature")
+# print(str(federated_algorithm.initialize.type_signature)
+# print("federated_algorithm.next.type_signature")
+# print(str(federated_algorithm.next.type_signature))
+
+# print("## Training the model")
+# iterative_process = tff.learning.build_federated_averaging_process(
+#     MnistModel,
+#     client_optimizer_fn=lambda: tf.keras.optimizers.SGD(learning_rate=0.02))
+
+# state = iterative_process.initialize()
+
+# for round_num in range(1, 11):
+#   state, metrics = iterative_process.next(state, federated_train_data)
+#   print('round {:2d}, metrics={}'.format(round_num, metrics))
+#evaluation = tff.learning.build_federated_evaluation(MnistModel)
+
+#TODO integration
+print("## Evaluation of the model")
\ No newline at end of file

src/participation-hub/federated-learning-microservice/app/processing/prototype/model_mnist_OLD.py

+import numpy as np
+import tensorflow as tf
+import tensorflow_federated as tff
+import collections
+
+input_spec_data_global = None
+
+MnistVariables = collections.namedtuple(
+    'MnistVariables', 'weights bias num_examples loss_sum accuracy_sum')
+
+def create_mnist_variables():
+  return MnistVariables(
+      weights=tf.Variable(
+          lambda: tf.zeros(dtype=tf.float32, shape=(784, 10)),
+          name='weights',
+          trainable=True),
+      bias=tf.Variable(
+          lambda: tf.zeros(dtype=tf.float32, shape=(10)),
+          name='bias',
+          trainable=True),
+      num_examples=tf.Variable(0.0, name='num_examples', trainable=False),
+      loss_sum=tf.Variable(0.0, name='loss_sum', trainable=False),
+      accuracy_sum=tf.Variable(0.0, name='accuracy_sum', trainable=False))
+
+def mnist_forward_pass(variables, batch):
+  y = tf.nn.softmax(tf.matmul(batch['x'], variables.weights) + variables.bias)
+  predictions = tf.cast(tf.argmax(y, 1), tf.int32)
+
+  flat_labels = tf.reshape(batch['y'], [-1])
+  loss = -tf.reduce_mean(
+      tf.reduce_sum(tf.one_hot(flat_labels, 10) * tf.math.log(y), axis=[1]))
+  accuracy = tf.reduce_mean(
+      tf.cast(tf.equal(predictions, flat_labels), tf.float32))
+
+  num_examples = tf.cast(tf.size(batch['y']), tf.float32)
+
+  variables.num_examples.assign_add(num_examples)
+  variables.loss_sum.assign_add(loss * num_examples)
+  variables.accuracy_sum.assign_add(accuracy * num_examples)
+
+  return loss, predictions
+
+def get_local_mnist_metrics(variables):
+  return collections.OrderedDict(
+      num_examples=variables.num_examples,
+      loss=variables.loss_sum / variables.num_examples,
+      accuracy=variables.accuracy_sum / variables.num_examples)
+
+@tff.federated_computation
+def aggregate_mnist_metrics_across_clients(metrics):
+  return collections.OrderedDict(
+      num_examples=tff.federated_sum(metrics.num_examples),
+      loss=tff.federated_mean(metrics.loss, metrics.num_examples),
+      accuracy=tff.federated_mean(metrics.accuracy, metrics.num_examples))
+
+class MnistModel(tff.learning.Model):
+
+  def __init__(self):
+    self._variables = create_mnist_variables()
+
+  @property
+  def trainable_variables(self):
+    return [self._variables.weights, self._variables.bias]
+
+  @property
+  def non_trainable_variables(self):
+    return []
+
+  @property
+  def local_variables(self):
+    return [
+        self._variables.num_examples, self._variables.loss_sum,
+        self._variables.accuracy_sum
+    ]
+
+  @property
+  def input_spec(self):
+    return collections.OrderedDict(
+        x=tf.TensorSpec([None, 784], tf.float32),
+        y=tf.TensorSpec([None, 1], tf.int32))
+
+    
+
+  @tf.function
+  def forward_pass(self, batch, training=True):
+    del training
+    loss, predictions = mnist_forward_pass(self._variables, batch)
+    num_exmaples = tf.shape(batch['x'])[0]
+    return tff.learning.BatchOutput(
+        loss=loss, predictions=predictions, num_examples=num_exmaples)
+
+  @tf.function
+  def report_local_outputs(self):
+    return get_local_mnist_metrics(self._variables)
+
+  @property
+  def federated_output_computation(self):
+    return aggregate_mnist_metrics_across_clients
\ No newline at end of file

src/participation-hub/federated-learning-microservice/app/processing/prototype/models2.py

+import numpy as np
+import tensorflow as tf
+import tensorflow_federated as tff
+import collections
+
+def create_keras_model(): #### DEFAULT TEST MODEL
+  return tf.keras.models.Sequential([
+      tf.keras.layers.InputLayer(input_shape=(784,)),
+      tf.keras.layers.Dense(10, kernel_initializer='zeros'),
+      tf.keras.layers.Softmax(),
+  ])
+
+
+def model_fn():
+  # We _must_ create a new model here, and _not_ capture it from an external
+  # scope. TFF will call this within different graph contexts.
+  keras_model = create_keras_model() ###TODO CAN BE CHANGED TO INCLUDE CALLS TO OTHER MODELS (or it cannot, because you cannot specify which model you want to call?)
+  return tff.learning.from_keras_model(
+      keras_model,
+      input_spec=get_input_spec(),
+      loss=tf.keras.losses.SparseCategoricalCrossentropy(),
+      metrics=[tf.keras.metrics.SparseCategoricalAccuracy()])
+
+
+def get_input_spec():
+    return collections.OrderedDict(
+        x=tf.TensorSpec([None, 784], tf.float32),
+        y=tf.TensorSpec([None, 1], tf.int32))
\ No newline at end of file

src/participation-hub/federated-learning-microservice/app/processing/text_processing/global_hyperparams.py

+
+def initialize():
+
+    global MAX_LENGTH #Lenght of sentences to be fed into the NN. Similar to image size i.e. 100pixels x 100pixels, but it's 1D.
+    MAX_LENGTH = 40
+    global VOCAB_SIZE #tells how many words it memorises. each word is stored as an int. 
+    VOCAB_SIZE = 6000
+    global NUM_CLIENTS #number of clients in the federated dataset
+    NUM_CLIENTS = 4
+    global SHUFFLE_BUFFER 
+    SHUFFLE_BUFFER = 5000 #used in preprocessing
+    global BATCH_SIZE
+    BATCH_SIZE = 512 #size of the batch of the dataset. which is later fed into the NN.
+    global INPUT_SPEC #tell the model how the data will look like.
+    INPUT_SPEC = None #must & will be initialized after data preprocessing. Currently it's being initialised with: train_dataset[0].element_spec
+    global EMBED_DIM # number of dimension of the embedding of the layer in the model.
+    EMBED_DIM = 10 
+    global EPOCHS #number of epochs the model will be trained
+    EPOCHS = 5 
\ No newline at end of file

src/participation-hub/federated-learning-microservice/app/processing/text_processing/main_proc.py

+
+import processing.text_processing.global_hyperparams as globals
+
+if __name__ == "__main__":
+
+    #globals.initialize()

src/participation-hub/federated-learning-microservice/app/processing/text_processing/preprocessing.py

+
+
+import pandas as pd
+
+from sklearn.model_selection import train_test_split
+from tensorflow.keras.models import Sequential
+from tensorflow.keras.layers import LSTM, Dense, Dropout, Embedding
+from tensorflow.keras.preprocessing.text import Tokenizer
+from tensorflow.keras.preprocessing.sequence import pad_sequences
+import collections
+import numpy as np
+import tensorflow as tf
+import tensorflow_federated as tff
+
+real = pd.read_csv("processing/fake_news/prototype_db_fake_real/True.csv")
+fake = pd.read_csv("processing/fake_news/prototype_db_fake_real/Fake.csv")
+
+# dropping rows that have urls as text and date, real's dates look fine, also dropping ones that have no text
+fake_drop = fake.drop(index=[9358,15507,15508,18933])
+fake_drop = fake_drop.drop(fake_drop.loc[fake_drop.text == ' '].index)
+real_drop = real.drop(real.loc[real.text == ' '].index)
+
+# Give labels to data before combining
+fake['label'] = 1
+real['label'] = 0
+combined = pd.concat([fake, real])
+
+no_reuters = combined.copy()
+no_reuters.text = no_reuters.text.str.replace('Reuters', '')
+combined = no_reuters.copy()
+## train/test split the text data and labels
+
+df_text = combined['text'] #features is now
+labels = combined['label'] #or maybe use target?
+target = combined['label'].values
+
+tokenizer = Tokenizer(oov_token = "<OOV>", num_words=6000)
+tokenizer.fit_on_texts(df_text)
+
+MAX_LENGTH = 40
+VOCAB_SIZE = 6000
+
+sequences_train = tokenizer.texts_to_sequences(df_text)
+
+padded_train = pad_sequences(sequences_train, padding = 'post', maxlen=MAX_LENGTH)
+
+#Data_train, data_text, label_train, label_test
+X_train, X_test, y_train, y_test = train_test_split(padded_train, target, test_size=0.2)
+
+X_train = tf.convert_to_tensor(X_train)
+X_test = tf.convert_to_tensor(X_test)
+y_train = tf.convert_to_tensor(y_train)
+y_test = tf.convert_to_tensor(y_test)

src/participation-hub/federated-learning-microservice/app/processing/text_processing/prototype_fed_text.py

+# import pandas as pd
+
+# train_df = pd.read_csv('processing/fake-news/train.csv', header=0)
+# test_df = pd.read_csv('processing/fake-news/test.csv', header=0)
+
+# train_df = train_df.fillna(' ')
+# test_df = test_df.fillna(' ')
+
+# train_df['all_info'] = train_df['text'] + train_df['title'] + train_df['author']
+# test_df['all_info'] = test_df['text'] + test_df['title'] + test_df['author']
+
+# target = train_df['label'].values
+
+
+
+import pandas as pd
+
+from sklearn.model_selection import train_test_split
+from tensorflow.keras.models import Sequential
+from tensorflow.keras.layers import LSTM, Dense, Dropout, Embedding
+from tensorflow.keras.preprocessing.text import Tokenizer
+from tensorflow.keras.preprocessing.sequence import pad_sequences
+import collections
+import numpy as np
+import tensorflow as tf
+import tensorflow_federated as tff
+
+real = pd.read_csv("processing/fake_news/prototype_db_fake_real/True.csv")
+fake = pd.read_csv("processing/fake_news/prototype_db_fake_real/Fake.csv")
+
+# dropping rows that have urls as text and date, real's dates look fine, also dropping ones that have no text
+fake_drop = fake.drop(index=[9358,15507,15508,18933])
+fake_drop = fake_drop.drop(fake_drop.loc[fake_drop.text == ' '].index)
+real_drop = real.drop(real.loc[real.text == ' '].index)
+
+# Give labels to data before combining
+fake['label'] = 1
+real['label'] = 0
+combined = pd.concat([fake, real])
+
+no_reuters = combined.copy()
+no_reuters.text = no_reuters.text.str.replace('Reuters', '')
+combined = no_reuters.copy()
+## train/test split the text data and labels
+
+df_text = combined['text'] #features is now
+labels = combined['label'] #or maybe use target?
+target = combined['label'].values
+
+print("##################label")
+print(type(labels))
+print(labels)
+print("###########")
+print(type(combined['label'].values))
+print(combined['label'].values)
+
+print("df_text_type:")
+print(type(df_text))
+############################ ^ORIGINAL DB
+
+
+# train_df = pd.read_csv('processing/text_processing/prototype_db_fake_real/train.csv', header=0)
+# test_df = pd.read_csv('processing/text_processing/prototype_db_fake_real/test.csv', header=0)
+
+# train_df = train_df.fillna(' ')
+# test_df = test_df.fillna(' ')
+
+# train_df['all_info'] = train_df['text'] + train_df['title'] + train_df['author']
+# test_df['all_info'] = test_df['text'] + test_df['title'] + test_df['author']
+
+# target = train_df['label'].values
+# print(type(train_df['label'].values))
+# print(train_df['label'].values)
+
+# df_text = train_df['all_info']
+
+######################################################################################
+tokenizer = Tokenizer(oov_token = "<OOV>", num_words=6000)
+tokenizer.fit_on_texts(df_text)
+
+MAX_LENGTH = 40
+VOCAB_SIZE = 6000
+
+sequences_train = tokenizer.texts_to_sequences(df_text)
+
+padded_train = pad_sequences(sequences_train, padding = 'post', maxlen=MAX_LENGTH)
+
+#Data_train, data_text, label_train, label_test
+X_train, X_test, y_train, y_test = train_test_split(padded_train, target, test_size=0.2)
+
+X_train = tf.convert_to_tensor(X_train)
+X_test = tf.convert_to_tensor(X_test)
+y_train = tf.convert_to_tensor(y_train)
+y_test = tf.convert_to_tensor(y_test)
+
+print(X_train.shape)
+print(y_train.shape)
+
+print("Type of X_train, X_test, y_train, y_test")
+print(type(X_train))
+print(type(X_test))
+print(type(y_train))
+print(type(y_test))
+###################################################################################\
+
+
+#FED PREPROCESSING
+
+NUM_CLIENTS = 4
+SHUFFLE_BUFFER = 5000
+BATCH_SIZE = 512
+
+def preprocess(dataset):
+  def element_fn(x, y):
+    return collections.OrderedDict([
+        ('x', x),
+        ('y', y)#tf.cast(tf.reshape(y, [1]), tf.float32))
+    ])
+
+  return dataset.map(element_fn).shuffle(
+      SHUFFLE_BUFFER).batch(BATCH_SIZE)
+
+def generate_clients_datasets(n, source_x, source_y):
+    clients_dataset=[]
+    for i in range(n):
+        dataset=tf.data.Dataset.from_tensor_slices(([source_x[i]], [source_y[i]]))
+        dataset=preprocess(dataset)
+        clients_dataset.append(dataset)
+    return clients_dataset
+
+train_dataset=generate_clients_datasets(NUM_CLIENTS, X_train, y_train)
+test_dataset=generate_clients_datasets(NUM_CLIENTS, X_test, y_test)
+
+
+# Grab a single batch of data so that TFF knows what data looks like.
+# sample_batch = tf.nest.map_structure(
+#     lambda x: x.numpy(), iter(train_dataset[0]).next())
+
+INPUT_SPEC = train_dataset[0].element_spec
+print("DONE PREPROCESSING")
+
+
+
+#################################################################################
+EMBED_DIM = 10
+
+def get_simple_LSTM_model():
+  model = Sequential()
+  model.add(Embedding(VOCAB_SIZE, EMBED_DIM, input_length=MAX_LENGTH))
+  model.add(Dropout(0.3))
+  model.add(LSTM(100))
+  model.add(Dropout(0.3))
+  model.add(Dense(64, activation='relu'))
+  model.add(Dropout(0.3))
+  model.add(Dense(1, activation='sigmoid'))
+
+#   model.compile(loss='binary_crossentropy',
+#               optimizer='adam', 
+#               metrics=[tf.keras.metrics.Precision(), tf.keras.metrics.Recall()])
+  return model
+
+
+def model_fn():
+  keras_model = get_simple_LSTM_model()
+  #return tff.learning.from_compiled_keras_model(keras_model, sample_batch) original
+
+  return tff.learning.from_keras_model(
+      keras_model,
+      input_spec=INPUT_SPEC,
+      loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
+      metrics=[tf.keras.metrics.SparseCategoricalAccuracy()])
+
+# Training and evaluating the model
+iterative_process = tff.learning.build_federated_averaging_process(model_fn,client_optimizer_fn=lambda: tf.keras.optimizers.SGD(lr=0.5))
+state = iterative_process.initialize()
+
+EPOCHS = 5
+
+for n in range(EPOCHS):
+    state, metrics = iterative_process.next(state, train_dataset)
+    print('round  {}, training metrics={}'.format(n+1, metrics))
+
+evaluation = tff.learning.build_federated_evaluation(model_fn)
+eval_metrics = evaluation(state.model, train_dataset)
+print('Training evaluation metrics={}'.format(eval_metrics))
+
+test_metrics = evaluation(state.model, test_dataset)
+print('Test evaluation metrics={}'.format(test_metrics))
+
+# model = get_simple_LSTM_model()
+# print(model.summary())
+
+
+# best_model_file_name = "processing/text_processing/models/best_model_LSTM.hdf5"
+# callbacks=[
+#     tf.keras.callbacks.EarlyStopping(monitor="val_loss", patience=15, 
+#                                   verbose=1, mode="min", restore_best_weights=True),
+#     tf.keras.callbacks.ModelCheckpoint(filepath=best_model_file_name, verbose=1, save_best_only=True)
+# ]
+
+# history = model.fit(X_train, 
+#                     y_train, 
+#                     epochs=EPOCHS, 
+#                     validation_data=(X_test, y_test), 
+#                     callbacks=callbacks)
+
+# model.save(best_model_file_name)
+# model = tf.keras.models.load_model(best_model_file_name)
+

src/participation-hub/federated-learning-microservice/app/processing/text_processing/prototype_text_LSTM_nonfed.py

+# import pandas as pd
+
+# train_df = pd.read_csv('processing/fake-news/train.csv', header=0)
+# test_df = pd.read_csv('processing/fake-news/test.csv', header=0)
+
+# train_df = train_df.fillna(' ')
+# test_df = test_df.fillna(' ')
+
+# train_df['all_info'] = train_df['text'] + train_df['title'] + train_df['author']
+# test_df['all_info'] = test_df['text'] + test_df['title'] + test_df['author']
+
+# target = train_df['label'].values
+
+
+
+import pandas as pd
+import re
+import matplotlib.pyplot as plt
+
+from sklearn.model_selection import train_test_split
+from sklearn.preprocessing import LabelEncoder
+from tensorflow.keras.models import Model
+from tensorflow.keras.models import Sequential
+from tensorflow.keras.layers import LSTM, Activation, Dense, Dropout, Input, Embedding
+from tensorflow.keras.optimizers import RMSprop
+from tensorflow.keras.preprocessing.text import Tokenizer
+from tensorflow.keras.preprocessing import sequence
+from tensorflow.keras.callbacks import EarlyStopping
+from tensorflow.keras.preprocessing.sequence import pad_sequences
+from nltk import word_tokenize
+import collections
+import numpy as np
+import tensorflow as tf
+import tensorflow_federated as tff
+
+real = pd.read_csv("processing/fake_news/prototype_db_fake_real/True.csv")
+fake = pd.read_csv("processing/fake_news/prototype_db_fake_real/Fake.csv")
+
+# dropping rows that have urls as text and date, real's dates look fine, also dropping ones that have no text
+fake_drop = fake.drop(index=[9358,15507,15508,18933])
+fake_drop = fake_drop.drop(fake_drop.loc[fake_drop.text == ' '].index)
+real_drop = real.drop(real.loc[real.text == ' '].index)
+
+# Give labels to data before combining
+fake['label'] = 1
+real['label'] = 0
+combined = pd.concat([fake, real])
+
+no_reuters = combined.copy()
+no_reuters.text = no_reuters.text.str.replace('Reuters', '')
+combined = no_reuters.copy()
+## train/test split the text data and labels
+
+train_df_text = combined['text'] #features is now
+labels = combined['label'] #or maybe use target?
+target = combined['label'].values
+
+print("##################label")
+print(type(labels))
+print(labels)
+print("###########")
+print(type(combined['label'].values))
+print(combined['label'].values)
+
+print("train_df_type:")
+print(type(train_df_text))
+############################ ^ORIGINAL DB
+
+
+# train_df = pd.read_csv('processing/text_processing/prototype_db_fake_real/train.csv', header=0)
+# test_df = pd.read_csv('processing/text_processing/prototype_db_fake_real/test.csv', header=0)
+
+# train_df = train_df.fillna(' ')
+# test_df = test_df.fillna(' ')
+
+# train_df['all_info'] = train_df['text'] + train_df['title'] + train_df['author']
+# test_df['all_info'] = test_df['text'] + test_df['title'] + test_df['author']
+
+# target = train_df['label'].values
+# print(type(train_df['label'].values))
+# print(train_df['label'].values)
+
+# train_df_text = train_df['all_info']
+
+######################################################################################
+tokenizer = Tokenizer(oov_token = "<OOV>", num_words=6000)
+tokenizer.fit_on_texts(train_df_text)
+
+max_length = 40
+vocab_size = 6000
+
+sequences_train = tokenizer.texts_to_sequences(train_df_text)
+
+padded_train = pad_sequences(sequences_train, padding = 'post', maxlen=max_length)
+
+X_train, X_test, y_train, y_test = train_test_split(padded_train, target, test_size=0.2)
+
+print(X_train.shape)
+print(y_train.shape)
+
+#################################################################################
+embed_dim = 10
+
+def get_simple_LSTM_model():
+  model = Sequential()
+  model.add(Embedding(vocab_size, embed_dim, input_length=max_length))
+  model.add(Dropout(0.3))
+  model.add(LSTM(100))
+  model.add(Dropout(0.3))
+  model.add(Dense(64, activation='relu'))
+  model.add(Dropout(0.3))
+  model.add(Dense(1, activation='sigmoid'))
+  return model
+
+model = get_simple_LSTM_model()
+print(model.summary())
+
+
+best_model_file_name = "processing/text_processing/models/best_model_LSTM.hdf5"
+callbacks=[
+    tf.keras.callbacks.EarlyStopping(monitor="val_loss", patience=15, 
+                                  verbose=1, mode="min", restore_best_weights=True),
+    tf.keras.callbacks.ModelCheckpoint(filepath=best_model_file_name, verbose=1, save_best_only=True)
+]
+
+model.compile(loss='binary_crossentropy',
+              optimizer='adam', 
+              metrics=[tf.keras.metrics.Precision(), tf.keras.metrics.Recall()])
+
+history = model.fit(X_train, 
+                    y_train, 
+                    epochs=5, 
+                    validation_data=(X_test, y_test), 
+                    callbacks=callbacks)
+
+model.save(best_model_file_name)
+model = tf.keras.models.load_model(best_model_file_name)
+

src/participation-hub/federated-learning-microservice/app/slurm-1780.out

+2021-05-05 12:34:15.886296: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
+Traceback (most recent call last):
+  File "processing/main_processing.py", line 11, in <module>
+    from federated_training_algorithm import get_federated_algorithm
+  File "/home/itec/bogdan/Articonf/smart/src/participation-hub/federated-learning-microservice/app/processing/federated_training_algorithm.py", line 48, in <module>
+    def server_init():
+  File "/home/itec/bogdan/.conda/envs/MLF/lib/python3.8/site-packages/tensorflow_federated/python/core/impl/wrappers/computation_wrapper.py", line 407, in __call__
+    result = fn_to_wrap(*args, **kwargs)
+  File "/home/itec/bogdan/Articonf/smart/src/participation-hub/federated-learning-microservice/app/processing/federated_training_algorithm.py", line 49, in server_init
+    model = model_fn()
+  File "/home/itec/bogdan/Articonf/smart/src/participation-hub/federated-learning-microservice/app/processing/models2.py", line 16, in model_fn
+    return tff.learning.from_keras_model(
+NameError: name 'tff' is not defined

src/participation-hub/federated-learning-microservice/app/slurm-1781.out

+2021-05-05 12:36:34.308397: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
+Traceback (most recent call last):
+  File "processing/main_processing.py", line 11, in <module>
+    from federated_training_algorithm import get_federated_algorithm
+  File "/home/itec/bogdan/Articonf/smart/src/participation-hub/federated-learning-microservice/app/processing/federated_training_algorithm.py", line 48, in <module>
+    def server_init():
+  File "/home/itec/bogdan/.conda/envs/MLF/lib/python3.8/site-packages/tensorflow_federated/python/core/impl/wrappers/computation_wrapper.py", line 407, in __call__
+    result = fn_to_wrap(*args, **kwargs)
+  File "/home/itec/bogdan/Articonf/smart/src/participation-hub/federated-learning-microservice/app/processing/federated_training_algorithm.py", line 49, in server_init
+    model = model_fn()
+  File "/home/itec/bogdan/Articonf/smart/src/participation-hub/federated-learning-microservice/app/processing/models2.py", line 19, in model_fn
+    input_spec=preprocessed_example_dataset.element_spec,
+NameError: name 'preprocessed_example_dataset' is not defined

src/participation-hub/federated-learning-microservice/app/slurm-1782.out

+2021-05-05 12:43:03.812706: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
+Traceback (most recent call last):
+  File "processing/main_processing.py", line 11, in <module>
+    from federated_training_algorithm import get_federated_algorithm
+  File "/home/itec/bogdan/Articonf/smart/src/participation-hub/federated-learning-microservice/app/processing/federated_training_algorithm.py", line 48, in <module>
+    def server_init():
+  File "/home/itec/bogdan/.conda/envs/MLF/lib/python3.8/site-packages/tensorflow_federated/python/core/impl/wrappers/computation_wrapper.py", line 407, in __call__
+    result = fn_to_wrap(*args, **kwargs)
+  File "/home/itec/bogdan/Articonf/smart/src/participation-hub/federated-learning-microservice/app/processing/federated_training_algorithm.py", line 49, in server_init
+    model = model_fn()
+  File "/home/itec/bogdan/Articonf/smart/src/participation-hub/federated-learning-microservice/app/processing/models2.py", line 19, in model_fn
+    input_spec=get_input_spec(),
+  File "/home/itec/bogdan/Articonf/smart/src/participation-hub/federated-learning-microservice/app/processing/models2.py", line 25, in get_input_spec
+    return collections.OrderedDict(
+NameError: name 'collections' is not defined

src/participation-hub/federated-learning-microservice/app/slurm-1783.out

+2021-05-05 12:44:48.174832: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
+2021-05-05 12:45:01.547785: W tensorflow/stream_executor/platform/default/dso_loader.cc:59] Could not load dynamic library 'libcuda.so.1'; dlerror: libcuda.so.1: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64
+2021-05-05 12:45:01.548090: W tensorflow/stream_executor/cuda/cuda_driver.cc:312] failed call to cuInit: UNKNOWN ERROR (303)
+2021-05-05 12:45:01.548116: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:156] kernel driver does not appear to be running on this host (mcore2): /proc/driver/nvidia/version does not exist
+2021-05-05 12:45:01.624528: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN)to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+2021-05-05 12:45:01.674208: I tensorflow/core/platform/profile_utils/cpu_utils.cc:104] CPU Frequency: 2599705000 Hz
+2021-05-05 12:45:01.676283: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x55727649d8f0 initialized for platform Host (this does not guarantee that XLA will be used). Devices:
+2021-05-05 12:45:01.676306: I tensorflow/compiler/xla/service/service.cc:176]   StreamExecutor device (0): Host, Default Version
+Traceback (most recent call last):
+  File "processing/main_processing.py", line 56, in <module>
+    print(merged_state[1])
+NameError: name 'merged_state' is not defined

src/participation-hub/federated-learning-microservice/app/slurm-1784.out

+2021-05-05 12:47:00.299027: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
+2021-05-05 12:47:14.153090: W tensorflow/stream_executor/platform/default/dso_loader.cc:59] Could not load dynamic library 'libcuda.so.1'; dlerror: libcuda.so.1: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64
+2021-05-05 12:47:14.153503: W tensorflow/stream_executor/cuda/cuda_driver.cc:312] failed call to cuInit: UNKNOWN ERROR (303)
+2021-05-05 12:47:14.153524: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:156] kernel driver does not appear to be running on this host (mcore2): /proc/driver/nvidia/version does not exist
+2021-05-05 12:47:14.220064: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN)to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+2021-05-05 12:47:14.250921: I tensorflow/core/platform/profile_utils/cpu_utils.cc:104] CPU Frequency: 2599705000 Hz
+2021-05-05 12:47:14.252596: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x55caa388f420 initialized for platform Host (this does not guarantee that XLA will be used). Devices:
+2021-05-05 12:47:14.252622: I tensorflow/compiler/xla/service/service.cc:176]   StreamExecutor device (0): Host, Default Version

src/participation-hub/federated-learning-microservice/app/slurm-1785.out

+2021-05-05 12:48:43.541178: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
+2021-05-05 12:48:57.548367: W tensorflow/stream_executor/platform/default/dso_loader.cc:59] Could not load dynamic library 'libcuda.so.1'; dlerror: libcuda.so.1: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64
+2021-05-05 12:48:57.548773: W tensorflow/stream_executor/cuda/cuda_driver.cc:312] failed call to cuInit: UNKNOWN ERROR (303)
+2021-05-05 12:48:57.548823: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:156] kernel driver does not appear to be running on this host (mcore2): /proc/driver/nvidia/version does not exist
+2021-05-05 12:48:57.617636: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN)to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+2021-05-05 12:48:57.658912: I tensorflow/core/platform/profile_utils/cpu_utils.cc:104] CPU Frequency: 2599705000 Hz
+2021-05-05 12:48:57.661020: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x55e17f536420 initialized for platform Host (this does not guarantee that XLA will be used). Devices:
+2021-05-05 12:48:57.661049: I tensorflow/compiler/xla/service/service.cc:176]   StreamExecutor device (0): Host, Default Version

src/participation-hub/federated-learning-microservice/app/slurm-1786.out

+2021-05-05 12:53:08.876262: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
+2021-05-05 12:53:22.787924: W tensorflow/stream_executor/platform/default/dso_loader.cc:59] Could not load dynamic library 'libcuda.so.1'; dlerror: libcuda.so.1: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64
+2021-05-05 12:53:22.788250: W tensorflow/stream_executor/cuda/cuda_driver.cc:312] failed call to cuInit: UNKNOWN ERROR (303)
+2021-05-05 12:53:22.788293: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:156] kernel driver does not appear to be running on this host (mcore2): /proc/driver/nvidia/version does not exist
+Traceback (most recent call last):
+  File "processing/main_processing.py", line 11, in <module>
+    from federated_training_algorithm import get_federated_algorithm
+  File "/home/itec/bogdan/Articonf/smart/src/participation-hub/federated-learning-microservice/app/processing/federated_training_algorithm.py", line 56, in <module>
+    whimsy_model = MnistModel()
+NameError: name 'MnistModel' is not defined

src/participation-hub/federated-learning-microservice/app/slurm-1787.out

+2021-05-05 12:55:42.203681: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
+2021-05-05 12:55:56.142028: W tensorflow/stream_executor/platform/default/dso_loader.cc:59] Could not load dynamic library 'libcuda.so.1'; dlerror: libcuda.so.1: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64
+2021-05-05 12:55:56.142493: W tensorflow/stream_executor/cuda/cuda_driver.cc:312] failed call to cuInit: UNKNOWN ERROR (303)
+2021-05-05 12:55:56.142516: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:156] kernel driver does not appear to be running on this host (mcore2): /proc/driver/nvidia/version does not exist
+2021-05-05 12:55:56.215449: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN)to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+2021-05-05 12:55:56.225527: I tensorflow/core/platform/profile_utils/cpu_utils.cc:104] CPU Frequency: 2599705000 Hz
+2021-05-05 12:55:56.227527: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x55be268bd420 initialized for platform Host (this does not guarantee that XLA will be used). Devices:
+2021-05-05 12:55:56.227548: I tensorflow/compiler/xla/service/service.cc:176]   StreamExecutor device (0): Host, Default Version

src/participation-hub/federated-learning-microservice/app/slurm-1788.out

+2021-05-05 13:51:03.378007: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
+Traceback (most recent call last):
+  File "processing/main_processing.py", line 10, in <module>
+    from models import MnistModel
+ModuleNotFoundError: No module named 'models'

src/participation-hub/federated-learning-microservice/app/slurm-1789.out

+2021-05-05 14:16:11.554269: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
+2021-05-05 14:16:28.615094: W tensorflow/stream_executor/platform/default/dso_loader.cc:59] Could not load dynamic library 'libcuda.so.1'; dlerror: libcuda.so.1: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64
+2021-05-05 14:16:28.615308: W tensorflow/stream_executor/cuda/cuda_driver.cc:312] failed call to cuInit: UNKNOWN ERROR (303)
+2021-05-05 14:16:28.615342: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:156] kernel driver does not appear to be running on this host (mcore2): /proc/driver/nvidia/version does not exist
+2021-05-05 14:16:28.696253: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN)to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+2021-05-05 14:16:28.725734: I tensorflow/core/platform/profile_utils/cpu_utils.cc:104] CPU Frequency: 2599705000 Hz
+2021-05-05 14:16:28.727680: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x56288f42e0d0 initialized for platform Host (this does not guarantee that XLA will be used). Devices:
+2021-05-05 14:16:28.727704: I tensorflow/compiler/xla/service/service.cc:176]   StreamExecutor device (0): Host, Default Version
+Traceback (most recent call last):
+  File "processing/main_processing.py", line 13, in <module>
+    from evaluation import evaluate
+  File "/home/itec/bogdan/Articonf/smart/src/participation-hub/federated-learning-microservice/app/processing/evaluation.py", line 4, in <module>
+    from models import MnistModel
+ModuleNotFoundError: No module named 'models'

src/participation-hub/federated-learning-microservice/app/slurm-1790.out

+2021-05-05 14:19:11.932075: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
+2021-05-05 14:19:25.830410: W tensorflow/stream_executor/platform/default/dso_loader.cc:59] Could not load dynamic library 'libcuda.so.1'; dlerror: libcuda.so.1: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64
+2021-05-05 14:19:25.830831: W tensorflow/stream_executor/cuda/cuda_driver.cc:312] failed call to cuInit: UNKNOWN ERROR (303)
+2021-05-05 14:19:25.830869: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:156] kernel driver does not appear to be running on this host (mcore2): /proc/driver/nvidia/version does not exist
+2021-05-05 14:19:25.902017: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN)to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+2021-05-05 14:19:25.911762: I tensorflow/core/platform/profile_utils/cpu_utils.cc:104] CPU Frequency: 2599705000 Hz
+2021-05-05 14:19:25.913382: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x563c6edbf450 initialized for platform Host (this does not guarantee that XLA will be used). Devices:
+2021-05-05 14:19:25.913401: I tensorflow/compiler/xla/service/service.cc:176]   StreamExecutor device (0): Host, Default Version

src/participation-hub/federated-learning-microservice/app/slurm-1791.out

+python: can't open file 'processing/main_processing.py': [Errno 2] No such file or directory