Skip to content

S
SMART
Commit c53b87b9 authored by Bogdan Mihai (ARTICONF student)'s avatar Bogdan Mihai (ARTICONF student)
Browse files
Options

Functional prototype fake news semi-fed model

parent 6d792c21
Hide whitespace changes
Inline Side-by-side
Showing with 230733 additions and 123 deletions
src/participation-hub/federated-learning-microservice/app/implementationLog
View file @ c53b87b9
## Starting...
## Preprocessing the federated_train_data
## Declaring the model
## Declaring the federated algorithm
tf_dataset_type
<x=float32[?,784],y=int32[?,1]>*
model_weights_type
<float32[784,10],float32[10]>
federated_algorithm.initialize.type_signature
( -> <float32[784,10],float32[10]>@SERVER)
federated_algorithm.next.type_signature
(<server_weights=<float32[784,10],float32[10]>@SERVER,federated_dataset={<x=float32[?,784],y=int32[?,1]>*}@CLIENTS> -> <float32[784,10],float32[10]>@SERVER)
## Training the model
round 1, metrics=OrderedDict([('broadcast', ()), ('aggregation', OrderedDict([('value_sum_process', ()), ('weight_sum_process', ())])), ('train', OrderedDict([('num_examples', 1023.0), ('loss', 2.9843163), ('accuracy', 0.086021505)]))])
round 2, metrics=OrderedDict([('broadcast', ()), ('aggregation', OrderedDict([('value_sum_process', ()), ('weight_sum_process', ())])), ('train', OrderedDict([('num_examples', 1023.0), ('loss', 2.9829092), ('accuracy', 0.088954054)]))])
round 3, metrics=OrderedDict([('broadcast', ()), ('aggregation', OrderedDict([('value_sum_process', ()), ('weight_sum_process', ())])), ('train', OrderedDict([('num_examples', 1023.0), ('loss', 2.9649642), ('accuracy', 0.092864126)]))])
round 4, metrics=OrderedDict([('broadcast', ()), ('aggregation', OrderedDict([('value_sum_process', ()), ('weight_sum_process', ())])), ('train', OrderedDict([('num_examples', 1023.0), ('loss', 2.937074), ('accuracy', 0.10068426)]))])
round 5, metrics=OrderedDict([('broadcast', ()), ('aggregation', OrderedDict([('value_sum_process', ()), ('weight_sum_process', ())])), ('train', OrderedDict([('num_examples', 1023.0), ('loss', 2.9075902), ('accuracy', 0.10166178)]))])
round 6, metrics=OrderedDict([('broadcast', ()), ('aggregation', OrderedDict([('value_sum_process', ()), ('weight_sum_process', ())])), ('train', OrderedDict([('num_examples', 1023.0), ('loss', 2.8781416), ('accuracy', 0.10459433)]))])
round 7, metrics=OrderedDict([('broadcast', ()), ('aggregation', OrderedDict([('value_sum_process', ()), ('weight_sum_process', ())])), ('train', OrderedDict([('num_examples', 1023.0), ('loss', 2.8490415), ('accuracy', 0.10557185)]))])
round 8, metrics=OrderedDict([('broadcast', ()), ('aggregation', OrderedDict([('value_sum_process', ()), ('weight_sum_process', ())])), ('train', OrderedDict([('num_examples', 1023.0), ('loss', 2.8203006), ('accuracy', 0.10752688)]))])
round 9, metrics=OrderedDict([('broadcast', ()), ('aggregation', OrderedDict([('value_sum_process', ()), ('weight_sum_process', ())])), ('train', OrderedDict([('num_examples', 1023.0), ('loss', 2.791923), ('accuracy', 0.11045943)]))])
round 10, metrics=OrderedDict([('broadcast', ()), ('aggregation', OrderedDict([('value_sum_process', ()), ('weight_sum_process', ())])), ('train', OrderedDict([('num_examples', 1023.0), ('loss', 2.7639053), ('accuracy', 0.11339199)]))])
## Evaluation of the model
## Starting...
## Preprocessing the federated_train_data
## Declaring the model
## Declaring the federated algorithm
tf_dataset_type
<x=float32[?,784],y=int32[?,1]>*
model_weights_type
<float32[784,10],float32[10]>
federated_algorithm.initialize.type_signature
( -> <float32[784,10],float32[10]>@SERVER)
federated_algorithm.next.type_signature
(<server_weights=<float32[784,10],float32[10]>@SERVER,federated_dataset={<x=float32[?,784],y=int32[?,1]>*}@CLIENTS> -> <float32[784,10],float32[10]>@SERVER)
## Training the model
round 1, metrics=OrderedDict([('broadcast', ()), ('aggregation', OrderedDict([('value_sum_process', ()), ('weight_sum_process', ())])), ('train', OrderedDict([('num_examples', 1023.0), ('loss', 2.9843163), ('accuracy', 0.086021505)]))])
round 2, metrics=OrderedDict([('broadcast', ()), ('aggregation', OrderedDict([('value_sum_process', ()), ('weight_sum_process', ())])), ('train', OrderedDict([('num_examples', 1023.0), ('loss', 2.982909), ('accuracy', 0.088954054)]))])
round 3, metrics=OrderedDict([('broadcast', ()), ('aggregation', OrderedDict([('value_sum_process', ()), ('weight_sum_process', ())])), ('train', OrderedDict([('num_examples', 1023.0), ('loss', 2.9649642), ('accuracy', 0.092864126)]))])
round 4, metrics=OrderedDict([('broadcast', ()), ('aggregation', OrderedDict([('value_sum_process', ()), ('weight_sum_process', ())])), ('train', OrderedDict([('num_examples', 1023.0), ('loss', 2.937074), ('accuracy', 0.10068426)]))])
round 5, metrics=OrderedDict([('broadcast', ()), ('aggregation', OrderedDict([('value_sum_process', ()), ('weight_sum_process', ())])), ('train', OrderedDict([('num_examples', 1023.0), ('loss', 2.9075902), ('accuracy', 0.10166178)]))])
round 6, metrics=OrderedDict([('broadcast', ()), ('aggregation', OrderedDict([('value_sum_process', ()), ('weight_sum_process', ())])), ('train', OrderedDict([('num_examples', 1023.0), ('loss', 2.8781416), ('accuracy', 0.10459433)]))])
round 7, metrics=OrderedDict([('broadcast', ()), ('aggregation', OrderedDict([('value_sum_process', ()), ('weight_sum_process', ())])), ('train', OrderedDict([('num_examples', 1023.0), ('loss', 2.849042), ('accuracy', 0.10557185)]))])
round 8, metrics=OrderedDict([('broadcast', ()), ('aggregation', OrderedDict([('value_sum_process', ()), ('weight_sum_process', ())])), ('train', OrderedDict([('num_examples', 1023.0), ('loss', 2.8203006), ('accuracy', 0.10752688)]))])
round 9, metrics=OrderedDict([('broadcast', ()), ('aggregation', OrderedDict([('value_sum_process', ()), ('weight_sum_process', ())])), ('train', OrderedDict([('num_examples', 1023.0), ('loss', 2.7919228), ('accuracy', 0.11045943)]))])
round 10, metrics=OrderedDict([('broadcast', ()), ('aggregation', OrderedDict([('value_sum_process', ()), ('weight_sum_process', ())])), ('train', OrderedDict([('num_examples', 1023.0), ('loss', 2.763905), ('accuracy', 0.11339199)]))])
## Evaluation of the model
## Starting...
## Preprocessing the federated_train_data
## Declaring the model
## Declaring the federated algorithm
tf_dataset_type
<x=float32[?,784],y=int32[?,1]>*
model_weights_type
<float32[784,10],float32[10]>
federated_algorithm.initialize.type_signature
( -> <float32[784,10],float32[10]>@SERVER)
federated_algorithm.next.type_signature
(<server_weights=<float32[784,10],float32[10]>@SERVER,federated_dataset={<x=float32[?,784],y=int32[?,1]>*}@CLIENTS> -> <float32[784,10],float32[10]>@SERVER)
## Training the model
round 1, metrics=OrderedDict([('broadcast', ()), ('aggregation', OrderedDict([('value_sum_process', ()), ('weight_sum_process', ())])), ('train', OrderedDict([('num_examples', 1023.0), ('loss', 2.9843163), ('accuracy', 0.086021505)]))])
round 2, metrics=OrderedDict([('broadcast', ()), ('aggregation', OrderedDict([('value_sum_process', ()), ('weight_sum_process', ())])), ('train', OrderedDict([('num_examples', 1023.0), ('loss', 2.982909), ('accuracy', 0.088954054)]))])
round 3, metrics=OrderedDict([('broadcast', ()), ('aggregation', OrderedDict([('value_sum_process', ()), ('weight_sum_process', ())])), ('train', OrderedDict([('num_examples', 1023.0), ('loss', 2.9649642), ('accuracy', 0.092864126)]))])
round 4, metrics=OrderedDict([('broadcast', ()), ('aggregation', OrderedDict([('value_sum_process', ()), ('weight_sum_process', ())])), ('train', OrderedDict([('num_examples', 1023.0), ('loss', 2.937074), ('accuracy', 0.10068426)]))])
round 5, metrics=OrderedDict([('broadcast', ()), ('aggregation', OrderedDict([('value_sum_process', ()), ('weight_sum_process', ())])), ('train', OrderedDict([('num_examples', 1023.0), ('loss', 2.9075902), ('accuracy', 0.10166178)]))])
round 6, metrics=OrderedDict([('broadcast', ()), ('aggregation', OrderedDict([('value_sum_process', ()), ('weight_sum_process', ())])), ('train', OrderedDict([('num_examples', 1023.0), ('loss', 2.8781419), ('accuracy', 0.10459433)]))])
round 7, metrics=OrderedDict([('broadcast', ()), ('aggregation', OrderedDict([('value_sum_process', ()), ('weight_sum_process', ())])), ('train', OrderedDict([('num_examples', 1023.0), ('loss', 2.8490417), ('accuracy', 0.10557185)]))])
round 8, metrics=OrderedDict([('broadcast', ()), ('aggregation', OrderedDict([('value_sum_process', ()), ('weight_sum_process', ())])), ('train', OrderedDict([('num_examples', 1023.0), ('loss', 2.8203006), ('accuracy', 0.10752688)]))])
round 9, metrics=OrderedDict([('broadcast', ()), ('aggregation', OrderedDict([('value_sum_process', ()), ('weight_sum_process', ())])), ('train', OrderedDict([('num_examples', 1023.0), ('loss', 2.7919226), ('accuracy', 0.11045943)]))])
round 10, metrics=OrderedDict([('broadcast', ()), ('aggregation', OrderedDict([('value_sum_process', ()), ('weight_sum_process', ())])), ('train', OrderedDict([('num_examples', 1023.0), ('loss', 2.7639048), ('accuracy', 0.11339199)]))])
## Evaluation of the model
## Starting...
## Preprocessing the federated_train_data
## Declaring the model
## Declaring the federated algorithm
## Training the model
round 1, metrics=OrderedDict([('broadcast', ()), ('aggregation', OrderedDict([('value_sum_process', ()), ('weight_sum_process', ())])), ('train', OrderedDict([('num_examples', 1023.0), ('loss', 2.9843166), ('accuracy', 0.086021505)]))])
round 2, metrics=OrderedDict([('broadcast', ()), ('aggregation', OrderedDict([('value_sum_process', ()), ('weight_sum_process', ())])), ('train', OrderedDict([('num_examples', 1023.0), ('loss', 2.9829092), ('accuracy', 0.088954054)]))])
round 3, metrics=OrderedDict([('broadcast', ()), ('aggregation', OrderedDict([('value_sum_process', ()), ('weight_sum_process', ())])), ('train', OrderedDict([('num_examples', 1023.0), ('loss', 2.964964), ('accuracy', 0.092864126)]))])
round 4, metrics=OrderedDict([('broadcast', ()), ('aggregation', OrderedDict([('value_sum_process', ()), ('weight_sum_process', ())])), ('train', OrderedDict([('num_examples', 1023.0), ('loss', 2.9370737), ('accuracy', 0.10068426)]))])
round 5, metrics=OrderedDict([('broadcast', ()), ('aggregation', OrderedDict([('value_sum_process', ()), ('weight_sum_process', ())])), ('train', OrderedDict([('num_examples', 1023.0), ('loss', 2.90759), ('accuracy', 0.10166178)]))])
round 6, metrics=OrderedDict([('broadcast', ()), ('aggregation', OrderedDict([('value_sum_process', ()), ('weight_sum_process', ())])), ('train', OrderedDict([('num_examples', 1023.0), ('loss', 2.8781416), ('accuracy', 0.10459433)]))])
round 7, metrics=OrderedDict([('broadcast', ()), ('aggregation', OrderedDict([('value_sum_process', ()), ('weight_sum_process', ())])), ('train', OrderedDict([('num_examples', 1023.0), ('loss', 2.8490417), ('accuracy', 0.10557185)]))])
round 8, metrics=OrderedDict([('broadcast', ()), ('aggregation', OrderedDict([('value_sum_process', ()), ('weight_sum_process', ())])), ('train', OrderedDict([('num_examples', 1023.0), ('loss', 2.8203006), ('accuracy', 0.10752688)]))])
round 9, metrics=OrderedDict([('broadcast', ()), ('aggregation', OrderedDict([('value_sum_process', ()), ('weight_sum_process', ())])), ('train', OrderedDict([('num_examples', 1023.0), ('loss', 2.7919228), ('accuracy', 0.11045943)]))])
round 10, metrics=OrderedDict([('broadcast', ()), ('aggregation', OrderedDict([('value_sum_process', ()), ('weight_sum_process', ())])), ('train', OrderedDict([('num_examples', 1023.0), ('loss', 2.763905), ('accuracy', 0.11339199)]))])
## Evaluation of the model
## Starting...
## Preprocessing the federated_train_data
## Declaring the model
## Declaring the federated algorithm
tf_dataset_type
<x=float32[?,784],y=int32[?,1]>*
model_weights_type
<float32[784,10],float32[10]>
FINISHEEED
## Evaluation of the model
## Starting...
## Preprocessing the federated_train_data
## Declaring the model
## Declaring the federated algorithm
tf_dataset_type
<x=float32[?,784],y=int32[?,1]>*
model_weights_type
<float32[784,10],float32[10]>
FINISHEEED
## Evaluation of the model
mean_client_wieghts
server_weights
<float32[784,10],float32[10]>@SERVER
mean_client_wieghts
<float32[784,10],float32[10]>@SERVER
......@@ -118,7 +10,22 @@ tf_dataset_type
<x=float32[?,784],y=int32[?,1]>*
model_weights_type
<float32[784,10],float32[10]>
server_state type
<class 'list'>
<class 'numpy.ndarray'>
FINISHEEED
server_state[1]
[ 5.4024562e-04 -1.6237081e-03 6.2275940e-04 1.4378619e-05
-3.4344319e-04 4.4040685e-04 -6.7906491e-05 -3.0773325e-04
1.3574951e-04 5.8925571e-04]
server2_state[1]
[ 5.8093132e-04 -2.8670396e-05 1.1061553e-04 -1.5197636e-04
-4.6668845e-04 2.7149473e-04 -1.8408171e-04 5.8942172e-05
-3.8304061e-04 1.9247324e-04]
merged_state[1]
[ 5.6058844e-04 -8.2618924e-04 3.6668745e-04 -6.8798872e-05
-4.0506583e-04 3.5595079e-04 -1.2599411e-04 -1.2439553e-04
-1.2364556e-04 3.9086447e-04]
## Evaluation of the model
server_weights
<float32[784,10],float32[10]>@SERVER
......@@ -132,21 +39,27 @@ tf_dataset_type
<x=float32[?,784],y=int32[?,1]>*
model_weights_type
<float32[784,10],float32[10]>
server_state type
<class 'list'>
<class 'numpy.ndarray'>
FINISHEEED
server_state[1]
[ 5.40245732e-04 -1.62370806e-03 6.22759399e-04 1.43785965e-05
-3.43443331e-04 4.40406700e-04 -6.79065852e-05 -3.07733397e-04
1.35749448e-04 5.89255709e-04]
server2_state[1]
[ 5.8093149e-04 -2.8670451e-05 1.1061558e-04 -1.5197645e-04
-4.6668845e-04 2.7149462e-04 -1.8408173e-04 5.8942172e-05
-3.8304055e-04 1.9247322e-04]
merged_state[1]
[ 5.6058861e-04 -8.2618924e-04 3.6668748e-04 -6.8798923e-05
-4.0506589e-04 3.5595067e-04 -1.2599415e-04 -1.2439561e-04
-1.2364556e-04 3.9086447e-04]
## Evaluation of the model
server_weights
<float32[784,10],float32[10]>@SERVER
mean_client_wieghts
<float32[784,10],float32[10]>@SERVER
## Starting...
## Preprocessing the federated_train_data
## Declaring the model
## Declaring the federated algorithm
tf_dataset_type
<x=float32[?,784],y=int32[?,1]>*
model_weights_type
<float32[784,10],float32[10]>
server_state
server_weights
<float32[784,10],float32[10]>@SERVER
mean_client_wieghts
......@@ -159,4 +72,20 @@ tf_dataset_type
<x=float32[?,784],y=int32[?,1]>*
model_weights_type
<float32[784,10],float32[10]>
server_state
server_state type
<class 'list'>
<class 'numpy.ndarray'>
FINISHEEED
server_state[1]
[ 5.4024585e-04 -1.6237078e-03 6.2275951e-04 1.4378643e-05
-3.4344324e-04 4.4040682e-04 -6.7906469e-05 -3.0773322e-04
1.3574972e-04 5.8925588e-04]
server2_state[1]
[ 5.80931432e-04 -2.86704162e-05 1.10615605e-04 -1.51976506e-04
-4.66688391e-04 2.71494675e-04 -1.84081669e-04 5.89420633e-05
-3.83040548e-04 1.92473250e-04]
merged_state[1]
[ 5.6058867e-04 -8.2618912e-04 3.6668757e-04 -6.8798930e-05
-4.0506583e-04 3.5595073e-04 -1.2599406e-04 -1.2439558e-04
-1.2364541e-04 3.9086456e-04]
## Evaluation of the model
src/participation-hub/federated-learning-microservice/app/mylog
View file @ c53b87b9
......@@ -50,3 +50,15 @@ Submitted batch job 1769
Submitted batch job 1777
Submitted batch job 1778
Submitted batch job 1779
Submitted batch job 1780
Submitted batch job 1781
Submitted batch job 1782
Submitted batch job 1783
Submitted batch job 1784
Submitted batch job 1785
Submitted batch job 1786
Submitted batch job 1787
Submitted batch job 1788
Submitted batch job 1789
Submitted batch job 1790
Submitted batch job 1791
src/participation-hub/federated-learning-microservice/app/processing/fake_news/aux_text.ipynb 0 → 100644
View file @ c53b87b9
{
"metadata": {
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.5"
},
"orig_nbformat": 2,
"kernelspec": {
"name": "python385jvsc74a57bd021aa800841c18c98a5f8e872010766fc499bdbd5453c920c0a4fd9c686874bbe",
"display_name": "Python 3.8.5 64-bit ('MLF': conda)"
},
"metadata": {
"interpreter": {
"hash": "21aa800841c18c98a5f8e872010766fc499bdbd5453c920c0a4fd9c686874bbe"
}
}
},
"nbformat": 4,
"nbformat_minor": 2,
"cells": [
{
"source": [
"import pandas as pd\r\n",
"import re\r\n",
"import matplotlib.pyplot as plt\r\n",
"\r\n",
"from sklearn.model_selection import train_test_split\r\n",
"from sklearn.preprocessing import LabelEncoder\r\n",
"from tensorflow.keras.models import Model\r\n",
"from tensorflow.keras.models import Sequential\r\n",
"from tensorflow.keras.layers import LSTM, Activation, Dense, Dropout, Input, Embedding\r\n",
"from tensorflow.keras.optimizers import RMSprop\r\n",
"from tensorflow.keras.preprocessing.text import Tokenizer\r\n",
"from tensorflow.keras.preprocessing import sequence\r\n",
"from tensorflow.keras.callbacks import EarlyStopping\r\n",
"from tensorflow.keras.preprocessing.sequence import pad_sequences\r\n",
"import collections\r\n",
"import numpy as np\r\n",
"import tensorflow as tf\r\n",
"import tensorflow_federated as tff\r\n"
],
"cell_type": "code",
"metadata": {},
"execution_count": 1,
"outputs": []
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"'\\nexperiment_name = \"mnist\"\\nmethod = \"tff_training\"\\nclient_lr = 1e-2\\nserver_lr = 1e-2\\nNUM_CLIENTS = 4\\nNUM_ROUNDS = 5\\nNUM_EPOCHS = 5\\nBATCH_SIZE = 20\\nPREFETCH_BUFFER = 10\\n\\nthis_dir = Path.cwd()\\nmodel_dir = this_dir / \"saved_models\" / experiment_name / method\\noutput_dir = this_dir / \"results\" / experiment_name / method\\n\\nif not model_dir.exists():\\n model_dir.mkdir(parents=True)\\n\\nif not output_dir.exists():\\n output_dir.mkdir(parents=True)\\n\\n(x_train, y_train), (x_test, y_test) = mnist.load_data()\\n\\nx_train = x_train.astype(np.float32)\\ny_train = y_train.astype(np.int32)\\nx_test = x_test.astype(np.float32).reshape(10000, 28, 28, 1)\\ny_test = y_test.astype(np.int32).reshape(10000, 1)\\n\\ntotal_image_count = len(x_train)\\nitems_per_set = int(np.floor(total_image_count/NUM_CLIENTS))\\n\\nclient_train_dataset = collections.OrderedDict()\\nfor i in range(1, NUM_CLIENTS+1):\\n client_name = \"client_\" + str(i)\\n start = items_per_set * (i-1)\\n end = items_per_set * i\\n\\n print(f\"Adding data from {start} to {end} for client : {client_name}\")\\n data = collections.OrderedDict(((\\'label\\', y_train[start:end]), (\\'pixels\\', x_train[start:end])))\\n client_train_dataset[client_name] = data\\n'"
]
},
"metadata": {},
"execution_count": 2
}
],
"source": [
"#########\n",
" ####\n",
"#token = Tokenizer(num_words=max_words, lower=True, split=' ')\n",
"#token.fit_on_texts(X_train.values)\n",
"#sequences = token.texts_to_sequences(X_train.values)\n",
"#train_sequences_padded = pad_sequences(sequences, maxlen=max_len)\n",
"\n",
"# def getTextModel2():\n",
"# model = Sequential()\n",
"# model.add(Embedding(max_words, embed_dim, input_length = max_len, input_shape=input_shape_var))\n",
"# return model\n",
"\n",
"####################################################################\n",
"# def getTextModel():\n",
"# model = Sequential()\n",
"# model.add(Embedding(max_words, embed_dim, input_length = max_len, input_shape=input_shape_var))\n",
"# model.add(LSTM(lstm_out))\n",
"# model.add(Dense(256))\n",
"# model.add(Activation('relu'))\n",
"# model.add(Dropout(0.5))\n",
"# model.add(Dense(1, name='out_layer'))\n",
"# model.add(Activation('sigmoid'))\n",
"# #model.compile(loss = 'binary_crossentropy', optimizer='adam',\\\n",
"# #metrics = ['accuracy'])\n",
"# return model\n",
"\n",
"# embed_dim = 50\n",
"# lstm_out = 64\n",
"# batch_size = 32\n",
"# input_shape_var = (max_words, )\n",
"# model = getTextModel()\n",
"# print(model.summary())\n",
"\n",
"# print()\n",
"\n",
"#TRAINING?\n",
"#history = model.fit(train_sequences_padded, y_train, batch_size=batch_size, epochs = 5, validation_split=0.2)\n",
"#test_sequences = token.texts_to_sequences(X_test)\n",
"#test_sequences_padded = pad_sequences(test_sequences, maxlen=max_len)\n",
"#model.evaluate(test_sequences_padded, y_test)\n",
"\n",
"\n",
"###################################################################\n",
"\"\"\"\n",
"experiment_name = \"mnist\"\n",
"method = \"tff_training\"\n",
"client_lr = 1e-2\n",
"server_lr = 1e-2\n",
"NUM_CLIENTS = 4\n",
"NUM_ROUNDS = 5\n",
"NUM_EPOCHS = 5\n",
"BATCH_SIZE = 20\n",
"PREFETCH_BUFFER = 10\n",
"\n",
"this_dir = Path.cwd()\n",
"model_dir = this_dir / \"saved_models\" / experiment_name / method\n",
"output_dir = this_dir / \"results\" / experiment_name / method\n",
"\n",
"if not model_dir.exists():\n",
" model_dir.mkdir(parents=True)\n",
"\n",
"if not output_dir.exists():\n",
" output_dir.mkdir(parents=True)\n",
"\n",
"(x_train, y_train), (x_test, y_test) = mnist.load_data()\n",
"\n",
"x_train = x_train.astype(np.float32)\n",
"y_train = y_train.astype(np.int32)\n",
"x_test = x_test.astype(np.float32).reshape(10000, 28, 28, 1)\n",
"y_test = y_test.astype(np.int32).reshape(10000, 1)\n",
"\n",
"total_image_count = len(x_train)\n",
"items_per_set = int(np.floor(total_image_count/NUM_CLIENTS))\n",
"\n",
"client_train_dataset = collections.OrderedDict()\n",
"for i in range(1, NUM_CLIENTS+1):\n",
" client_name = \"client_\" + str(i)\n",
" start = items_per_set * (i-1)\n",
" end = items_per_set * i\n",
"\n",
" print(f\"Adding data from {start} to {end} for client : {client_name}\")\n",
" data = collections.OrderedDict((('label', y_train[start:end]), ('pixels', x_train[start:end])))\n",
" client_train_dataset[client_name] = data\n",
"\"\"\"\n"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"<class 'pandas.core.series.Series'>\nAdding data from 0 to 8418 for client : client_1\nAdding data from 8418 to 16836 for client : client_2\nAdding data from 16836 to 25254 for client : client_3\nAdding data from 25254 to 33672 for client : client_4\n"
]
}
],
"source": [
"\n",
"#real = pd.read_csv(\"processing/fake_news/prototype_db_fake_real/True.csv\")\n",
"#fake = pd.read_csv(\"processing/fake_news/prototype_db_fake_real/Fake.csv\")\n",
"real = pd.read_csv(\"prototype_db_fake_real/True.csv\")\n",
"fake = pd.read_csv(\"prototype_db_fake_real/Fake.csv\")\n",
"\n",
"# dropping rows that have urls as text and date, real's dates look fine, also dropping ones that have no text\n",
"fake_drop = fake.drop(index=[9358,15507,15508,18933])\n",
"fake_drop = fake_drop.drop(fake_drop.loc[fake_drop.text == ' '].index)\n",
"real_drop = real.drop(real.loc[real.text == ' '].index)\n",
"\n",
"# Give labels to data before combining\n",
"fake['fake'] = 1\n",
"real['fake'] = 0\n",
"combined = pd.concat([fake, real])\n",
"\n",
"no_reuters = combined.copy()\n",
"no_reuters.text = no_reuters.text.str.replace('Reuters', '')\n",
"combined = no_reuters.copy()\n",
"## train/test split the text data and labels\n",
"features = combined['text']\n",
"labels = combined['fake']\n",
"X_train, X_test, y_train, y_test = train_test_split(features, labels, random_state = 42)\n",
"print(type(X_train))\n",
"# the model will remember only the top 2000 most common words\n",
"max_words = 2000\n",
"max_len = 400\n",
"NUM_CLIENTS = 4\n",
"\n",
" #######\n",
"#TODO\n",
"lenght_of_traindataset = len(X_train) #uita-te jos cum e facut\n",
"items_per_set = int(np.floor(lenght_of_traindataset/NUM_CLIENTS))\n",
"\n",
"client_train_dataset = collections.OrderedDict()\n",
"for i in range(1, NUM_CLIENTS+1):\n",
" client_name = \"client_\" + str(i)\n",
" start = items_per_set * (i-1)\n",
" end = items_per_set * i\n",
"\n",
" print(f\"Adding data from {start} to {end} for client : {client_name}\")\n",
" data = collections.OrderedDict((('label', y_train[start:end]), ('features', X_train[start:end]))) #features in this case is equivalent to 'text', in image_classifiers case, it's the 'pixels'\n",
" client_train_dataset[client_name] = data\n",
"\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"OrderedDict([('label', <tf.Tensor: shape=(), dtype=int64, numpy=0>), ('features', <tf.Tensor: shape=(), dtype=string, numpy=b'SEOUL () - South Korea predicted on Tuesday that North Korea would look to open negotiations with the United States next year in an optimistic outlook for 2018, even as Seoul set up a specialized military team to confront nuclear threats from the North. The U.N. Security Council unanimously imposed new, tougher sanctions on reclusive North Korea on Friday for its recent intercontinental ballistic missile test, a move the North branded an economic blockade and act of war. North Korea will seek negotiation with United States, while continuing to pursue its effort to be recognized as a de facto nuclear-possessing country, South Korea s Unification Ministry said in a report, without offering any reasons for its conclusion. The Ministry of Defence said it would assign four units to operate under a new official overseeing North Korea policy, aimed to deter and respond to North Korea s nuclear and missile threat . Tensions have risen over North Korea s nuclear and missile programs, which it pursues in defiance of years of U.N. Security Council resolutions, with bellicose rhetoric coming from both Pyongyang and the White House. U.S. diplomats have made clear they are seeking a diplomatic solution but President Donald Trump has derided talks as useless and said Pyongyang must commit to giving up its nuclear weapons before any talks can begin. In a statement carried by the official KCNA news agency, North Korea said the United States was terrified by its nuclear force and was getting more and more frenzied in the moves to impose the harshest-ever sanctions and pressure on our country . China, the North s lone major ally, and Russia both supported the latest U.N. sanctions, which seek to limit the North s access to refined petroleum products and crude oil and its earnings from workers abroad, while on Monday Chinese Foreign Ministry spokeswoman Hua Chunying called for all countries to ease tension. On Tuesday, Beijing released customs data indicating China exported no oil products to North Korea in November, apparently going over and beyond U.N. sanctions. China, the main source of North Korea s fuel, did not export any gasoline, jet fuel, diesel or fuel oil to its neighbor last month, data from the General Administration of Customs showed. China also imported no iron ore, coal or lead from North Korea in November. In its 2018 forecast, South Korea s Unification Ministry said it believed the North would eventually find ways to blunt the effects of the sanctions. Countermeasures will be orchestrated to deal with the effects, including cuts in trade volume and foreign currency inflow, lack of supplies, and reduced production in each part of the economy, the report said. The latest round of sanctions was prompted by the Nov. 29 test of what North Korea said was an intercontinental ballistic missile that put the U.S. mainland within range of its nuclear weapons. The Joongang Ilbo Daily newspaper, citing an unnamed South Korean government official, reported on Tuesday that North Korea could also be preparing to launch a satellite into space. Experts have said such launches are likely aimed at further developing the North s ballistic missile technology, and as such would be prohibited under U.N. resolutions. The North Korean Rodong Sinmun newspaper said on Monday saying that peaceful space development is a legitimate right of a sovereign state . North Korea regularly threatens to destroy South Korea, the United States and Japan, and says its weapons are necessary to counter U.S. aggression. The United States stations 28,500 troops in the South, a legacy of the 1950-53 Korean War, and regularly carries out military exercises with the South, which the North sees as preparations for invasion. '>)])\n"
]
}
],
"source": [
"train_dataset = tff.simulation.FromTensorSlicesClientData(client_train_dataset)\n",
"sample_dataset = train_dataset.create_tf_dataset_for_client(train_dataset.client_ids[0])\n",
"\n",
"@tf.function\n",
"def aux_fn(sample_dataset):\n",
" return next(iter(sample_dataset))\n",
"\n",
"sample_element = aux_fn(sample_dataset)\n",
"print(sample_element)"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"\n",
"PROCESSING BATCH\n",
"Tensor(\"args_0:0\", shape=(None,), dtype=string)\n",
"\n",
"PROCESSING BATCH\n",
"Tensor(\"args_0:0\", shape=(None,), dtype=string)\n",
"\n",
"PROCESSING BATCH\n",
"Tensor(\"args_0:0\", shape=(None,), dtype=string)\n",
"\n",
"PROCESSING BATCH\n",
"Tensor(\"args_0:0\", shape=(None,), dtype=string)\n",
"\n",
"PROCESSING BATCH\n",
"Tensor(\"args_0:0\", shape=(None,), dtype=string)\n",
"Number of client datasets: 4\n",
"First dataset: <PrefetchDataset shapes: OrderedDict([(x, (None,)), (y, (None, 1))]), types: OrderedDict([(x, tf.string), (y, tf.int64)])>\n"
]
}
],
"source": [
"\n",
"\n",
"SHUFFLE_BUFFER = items_per_set\n",
"#################TESTAT PANA AICI ^\n",
"\n",
"method = \"tff_training\"\n",
"client_lr = 1e-2\n",
"server_lr = 1e-2\n",
"NUM_CLIENTS = 4\n",
"NUM_ROUNDS = 5\n",
"NUM_EPOCHS = 5\n",
"BATCH_SIZE = 20\n",
"PREFETCH_BUFFER = 10\n",
"\n",
"def preprocess(dataset):\n",
"\n",
" def batch_format_fn(element):\n",
" \"\"\"Flatten a batch `pixels` and return the features as an `OrderedDict`.\"\"\"\n",
" #return element\n",
" print(\"\")\n",
" print(\"PROCESSING BATCH\")\n",
" print(element['features'])\n",
" return collections.OrderedDict(\n",
" x=element['features'],#tf.reshape(element['pixels'], [-1, 28, 28, 1]),\n",
" y=tf.reshape(element['label'], [-1, 1])\n",
" )\n",
" return dataset.repeat(NUM_EPOCHS).shuffle(SHUFFLE_BUFFER).batch(\n",
" BATCH_SIZE).map(batch_format_fn).prefetch(PREFETCH_BUFFER)\n",
"\n",
"preprocessed_sample_dataset = preprocess(sample_dataset)\n",
"#usless sample_batch = nest.map_structure(lambda x: x.numpy(), next(iter(preprocessed_sample_dataset)))\n",
"\n",
"\n",
"def make_federated_data(client_data, client_ids):\n",
" return [preprocess(client_data.create_tf_dataset_for_client(x)) for x in client_ids]\n",
"\n",
"federated_train_data = make_federated_data(train_dataset, train_dataset.client_ids)\n",
"\n",
"print('Number of client datasets: {l}'.format(l=len(federated_train_data)))\n",
"print('First dataset: {d}'.format(d=federated_train_data[0]))\n",
"\n",
"auxvar = preprocessed_sample_dataset.element_spec\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Create a custom standardization function to strip HTML break tags '<br />'.\n",
"def custom_standardization(input_data):\n",
" lowercase = tf.strings.lower(input_data)\n",
" stripped_html = tf.strings.regex_replace(lowercase, '<br />', ' ')\n",
" return tf.strings.regex_replace(stripped_html,\n",
" '[%s]' % re.escape(string.punctuation), '')\n",
"\n",
"\n",
"# Vocabulary size and number of words in a sequence.\n",
"vocab_size = 10000\n",
"sequence_length = 100\n",
"\n",
"# Use the text vectorization layer to normalize, split, and map strings to\n",
"# integers. Note that the layer uses the custom standardization defined above.\n",
"# Set maximum_sequence length as all samples are not of the same length.\n",
"vectorize_layer = TextVectorization(\n",
" standardize=custom_standardization,\n",
" max_tokens=vocab_size,\n",
" output_mode='int',\n",
" output_sequence_length=sequence_length)\n",
"\n",
"# Make a text-only dataset (no labels) and call adapt to build the vocabulary.\n",
"text_ds = train_ds.map(lambda x, y: x)\n",
"vectorize_layer.adapt(text_ds)"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"WARNING:tensorflow:Layer lstm_1 will not use cuDNN kernel since it doesn't meet the cuDNN kernel criteria. It will use generic GPU kernel as fallback when running on GPU\n",
"Model: \"sequential_1\"\n",
"_________________________________________________________________\n",
"Layer (type) Output Shape Param # \n",
"=================================================================\n",
"embedding_1 (Embedding) (None, None, 50) 100000 \n",
"_________________________________________________________________\n",
"lstm_1 (LSTM) (None, 64) 29440 \n",
"_________________________________________________________________\n",
"dense_1 (Dense) (None, 256) 16640 \n",
"_________________________________________________________________\n",
"activation_2 (Activation) (None, 256) 0 \n",
"_________________________________________________________________\n",
"dropout_1 (Dropout) (None, 256) 0 \n",
"_________________________________________________________________\n",
"out_layer (Dense) (None, 1) 257 \n",
"_________________________________________________________________\n",
"activation_3 (Activation) (None, 1) 0 \n",
"=================================================================\n",
"Total params: 146,337\n",
"Trainable params: 146,337\n",
"Non-trainable params: 0\n",
"_________________________________________________________________\n",
"None\n"
]
}
],
"source": [
"def getTextModel():\n",
" model = Sequential()\n",
" model.add(vectorize_layer)\n",
" model.add(Embedding(max_words, embed_dim)),#, input_length = max_len))#, input_shape=auxvar))\n",
" model.add(LSTM(lstm_out))\n",
" model.add(Dense(256))\n",
" model.add(Activation('relu'))\n",
" model.add(Dropout(0.5))\n",
" model.add(Dense(1, name='out_layer'))\n",
" model.add(Activation('sigmoid'))\n",
" #model.compile(loss = 'binary_crossentropy', optimizer='adam',\\\n",
" #metrics = ['accuracy'])\n",
" return model\n",
"\n",
"embed_dim = 50\n",
"lstm_out = 64\n",
"batch_size = 32\n",
"input_shape_var = (max_words, )\n",
"model = getTextModel()\n",
"print(model.summary())\n",
"\n",
"\n",
"\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"WARNING:tensorflow:Layer lstm will not use cuDNN kernel since it doesn't meet the cuDNN kernel criteria. It will use generic GPU kernel as fallback when running on GPU\n",
"WARNING:tensorflow:Layer lstm will not use cuDNN kernel since it doesn't meet the cuDNN kernel criteria. It will use generic GPU kernel as fallback when running on GPU\n",
"WARNING:tensorflow:Layer lstm will not use cuDNN kernel since it doesn't meet the cuDNN kernel criteria. It will use generic GPU kernel as fallback when running on GPU\n",
"WARNING:tensorflow:Layer lstm will not use cuDNN kernel since it doesn't meet the cuDNN kernel criteria. It will use generic GPU kernel as fallback when running on GPU\n",
"WARNING:tensorflow:Layer lstm will not use cuDNN kernel since it doesn't meet the cuDNN kernel criteria. It will use generic GPU kernel as fallback when running on GPU\n",
"WARNING:tensorflow:Model was constructed with shape (None, None) for input Tensor(\"embedding_input:0\", shape=(None, None), dtype=float32), but it was called on an input with incompatible shape (None,).\n"
]
},
{
"output_type": "error",
"ename": "ValueError",
"evalue": "in user code:\n\n /home/itec/bogdan/.conda/envs/MLF/lib/python3.8/site-packages/tensorflow_federated/python/learning/framework/optimizer_utils.py:387 _compute_local_training_and_client_delta *\n client_output = client_delta_fn(dataset, initial_model_weights)\n /home/itec/bogdan/.conda/envs/MLF/lib/python3.8/site-packages/tensorflow_federated/python/learning/federated_averaging.py:92 reduce_fn *\n output = model.forward_pass(batch, training=True)\n /home/itec/bogdan/.conda/envs/MLF/lib/python3.8/site-packages/tensorflow_federated/python/learning/framework/dataset_reduce.py:28 _dataset_reduce_fn *\n return dataset.reduce(initial_state=initial_state_fn(), reduce_func=reduce_fn)\n /home/itec/bogdan/.conda/envs/MLF/lib/python3.8/site-packages/tensorflow_federated/python/learning/keras_utils.py:365 forward_pass *\n return self._forward_pass(batch_input, training=training)\n /home/itec/bogdan/.conda/envs/MLF/lib/python3.8/site-packages/tensorflow_federated/python/learning/keras_utils.py:322 _forward_pass *\n predictions = self._keras_model(inputs, training=training)\n /home/itec/bogdan/.conda/envs/MLF/lib/python3.8/site-packages/tensorflow/python/keras/engine/base_layer_v1.py:776 __call__ **\n outputs = call_fn(cast_inputs, *args, **kwargs)\n /home/itec/bogdan/.conda/envs/MLF/lib/python3.8/site-packages/tensorflow/python/keras/engine/sequential.py:372 call\n return super(Sequential, self).call(inputs, training=training, mask=mask)\n /home/itec/bogdan/.conda/envs/MLF/lib/python3.8/site-packages/tensorflow/python/keras/engine/functional.py:385 call\n return self._run_internal_graph(\n /home/itec/bogdan/.conda/envs/MLF/lib/python3.8/site-packages/tensorflow/python/keras/engine/functional.py:508 _run_internal_graph\n outputs = node.layer(*args, **kwargs)\n /home/itec/bogdan/.conda/envs/MLF/lib/python3.8/site-packages/tensorflow/python/keras/layers/recurrent.py:663 __call__\n return super(RNN, self).__call__(inputs, **kwargs)\n /home/itec/bogdan/.conda/envs/MLF/lib/python3.8/site-packages/tensorflow/python/keras/engine/base_layer_v1.py:751 __call__\n input_spec.assert_input_compatibility(self.input_spec, inputs,\n /home/itec/bogdan/.conda/envs/MLF/lib/python3.8/site-packages/tensorflow/python/keras/engine/input_spec.py:176 assert_input_compatibility\n raise ValueError('Input ' + str(input_index) + ' of layer ' +\n\n ValueError: Input 0 of layer lstm is incompatible with the layer: expected ndim=3, found ndim=2. Full shape received: [None, 50]\n",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m<ipython-input-9-939251a78ed1>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m 12\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 13\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 14\u001b[0;31m iterative_process = tff.learning.build_federated_averaging_process(\n\u001b[0m\u001b[1;32m 15\u001b[0m \u001b[0mmodel_fn\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 16\u001b[0m \u001b[0mclient_optimizer_fn\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mlambda\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkeras\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptimizers\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mAdam\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlearning_rate\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mclient_lr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/.conda/envs/MLF/lib/python3.8/site-packages/tensorflow_federated/python/learning/federated_averaging.py\u001b[0m in \u001b[0;36mbuild_federated_averaging_process\u001b[0;34m(model_fn, client_optimizer_fn, server_optimizer_fn, client_weight_fn, broadcast_process, aggregation_process, model_update_aggregation_factory, use_experimental_simulation_loop)\u001b[0m\n\u001b[1;32m 221\u001b[0m use_experimental_simulation_loop)\n\u001b[1;32m 222\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 223\u001b[0;31m return optimizer_utils.build_model_delta_optimizer_process(\n\u001b[0m\u001b[1;32m 224\u001b[0m \u001b[0mmodel_fn\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 225\u001b[0m \u001b[0mmodel_to_client_delta_fn\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mclient_fed_avg\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/.conda/envs/MLF/lib/python3.8/site-packages/tensorflow_federated/python/learning/framework/optimizer_utils.py\u001b[0m in \u001b[0;36mbuild_model_delta_optimizer_process\u001b[0;34m(model_fn, model_to_client_delta_fn, server_optimizer_fn, broadcast_process, aggregation_process, model_update_aggregation_factory)\u001b[0m\n\u001b[1;32m 647\u001b[0m aggregation_process=aggregation_process)\n\u001b[1;32m 648\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 649\u001b[0;31m run_one_round_computation = _build_one_round_computation(\n\u001b[0m\u001b[1;32m 650\u001b[0m \u001b[0mmodel_fn\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmodel_fn\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 651\u001b[0m \u001b[0mserver_optimizer_fn\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mserver_optimizer_fn\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/.conda/envs/MLF/lib/python3.8/site-packages/tensorflow_federated/python/learning/framework/optimizer_utils.py\u001b[0m in \u001b[0;36m_build_one_round_computation\u001b[0;34m(model_fn, server_optimizer_fn, model_to_client_delta_fn, broadcast_process, aggregation_process)\u001b[0m\n\u001b[1;32m 372\u001b[0m \u001b[0;34m@\u001b[0m\u001b[0mcomputations\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtf_computation\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdataset_type\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel_weights_type\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 373\u001b[0m \u001b[0;34m@\u001b[0m\u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfunction\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 374\u001b[0;31m \u001b[0;32mdef\u001b[0m \u001b[0m_compute_local_training_and_client_delta\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minitial_model_weights\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 375\u001b[0m \"\"\"Performs client local model optimization.\n\u001b[1;32m 376\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/.conda/envs/MLF/lib/python3.8/site-packages/tensorflow_federated/python/core/impl/wrappers/computation_wrapper.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, tff_internal_types, *args)\u001b[0m\n\u001b[1;32m 405\u001b[0m parameter_type)\n\u001b[1;32m 406\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0munpack_arguments_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mwrapped_fn_generator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 407\u001b[0;31m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfn_to_wrap\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 408\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mresult\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 409\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mComputationReturnedNoneError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfn_to_wrap\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/.conda/envs/MLF/lib/python3.8/site-packages/tensorflow/python/eager/def_function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m 778\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 779\u001b[0m \u001b[0mcompiler\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"nonXla\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 780\u001b[0;31m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 781\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 782\u001b[0m \u001b[0mnew_tracing_count\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_tracing_count\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/.conda/envs/MLF/lib/python3.8/site-packages/tensorflow/python/eager/def_function.py\u001b[0m in \u001b[0;36m_call\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m 821\u001b[0m \u001b[0;31m# This is the first call of __call__, so we have to initialize.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 822\u001b[0m \u001b[0minitializers\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 823\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_initialize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0madd_initializers_to\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minitializers\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 824\u001b[0m \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 825\u001b[0m \u001b[0;31m# At this point we know that the initialization is complete (or less\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/.conda/envs/MLF/lib/python3.8/site-packages/tensorflow/python/eager/def_function.py\u001b[0m in \u001b[0;36m_initialize\u001b[0;34m(self, args, kwds, add_initializers_to)\u001b[0m\n\u001b[1;32m 694\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_graph_deleter\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mFunctionDeleter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_lifted_initializer_graph\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 695\u001b[0m self._concrete_stateful_fn = (\n\u001b[0;32m--> 696\u001b[0;31m self._stateful_fn._get_concrete_function_internal_garbage_collected( # pylint: disable=protected-access\n\u001b[0m\u001b[1;32m 697\u001b[0m *args, **kwds))\n\u001b[1;32m 698\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/.conda/envs/MLF/lib/python3.8/site-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m_get_concrete_function_internal_garbage_collected\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 2853\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2854\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_lock\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2855\u001b[0;31m \u001b[0mgraph_function\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_maybe_define_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2856\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mgraph_function\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2857\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/.conda/envs/MLF/lib/python3.8/site-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m_maybe_define_function\u001b[0;34m(self, args, kwargs)\u001b[0m\n\u001b[1;32m 3211\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3212\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_function_cache\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmissed\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcall_context_key\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3213\u001b[0;31m \u001b[0mgraph_function\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_create_graph_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 3214\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_function_cache\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprimary\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mcache_key\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgraph_function\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3215\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mgraph_function\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/.conda/envs/MLF/lib/python3.8/site-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m_create_graph_function\u001b[0;34m(self, args, kwargs, override_flat_arg_shapes)\u001b[0m\n\u001b[1;32m 3063\u001b[0m \u001b[0marg_names\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbase_arg_names\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mmissing_arg_names\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3064\u001b[0m graph_function = ConcreteFunction(\n\u001b[0;32m-> 3065\u001b[0;31m func_graph_module.func_graph_from_py_func(\n\u001b[0m\u001b[1;32m 3066\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_name\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3067\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_python_function\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/.conda/envs/MLF/lib/python3.8/site-packages/tensorflow/python/framework/func_graph.py\u001b[0m in \u001b[0;36mfunc_graph_from_py_func\u001b[0;34m(name, python_func, args, kwargs, signature, func_graph, autograph, autograph_options, add_control_dependencies, arg_names, op_return_value, collections, capture_by_value, override_flat_arg_shapes)\u001b[0m\n\u001b[1;32m 984\u001b[0m \u001b[0m_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moriginal_func\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_decorator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munwrap\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpython_func\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 985\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 986\u001b[0;31m \u001b[0mfunc_outputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpython_func\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mfunc_args\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mfunc_kwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 987\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 988\u001b[0m \u001b[0;31m# invariant: `func_outputs` contains only Tensors, CompositeTensors,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/.conda/envs/MLF/lib/python3.8/site-packages/tensorflow/python/eager/def_function.py\u001b[0m in \u001b[0;36mwrapped_fn\u001b[0;34m(*args, **kwds)\u001b[0m\n\u001b[1;32m 598\u001b[0m \u001b[0;31m# __wrapped__ allows AutoGraph to swap in a converted function. We give\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 599\u001b[0m \u001b[0;31m# the function a weak reference to itself to avoid a reference cycle.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 600\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mweak_wrapped_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__wrapped__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 601\u001b[0m \u001b[0mweak_wrapped_fn\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mweakref\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mref\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mwrapped_fn\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 602\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/.conda/envs/MLF/lib/python3.8/site-packages/tensorflow/python/framework/func_graph.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 971\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;31m# pylint:disable=broad-except\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 972\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"ag_error_metadata\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 973\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mag_error_metadata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_exception\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 974\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 975\u001b[0m \u001b[0;32mraise\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;31mValueError\u001b[0m: in user code:\n\n /home/itec/bogdan/.conda/envs/MLF/lib/python3.8/site-packages/tensorflow_federated/python/learning/framework/optimizer_utils.py:387 _compute_local_training_and_client_delta *\n client_output = client_delta_fn(dataset, initial_model_weights)\n /home/itec/bogdan/.conda/envs/MLF/lib/python3.8/site-packages/tensorflow_federated/python/learning/federated_averaging.py:92 reduce_fn *\n output = model.forward_pass(batch, training=True)\n /home/itec/bogdan/.conda/envs/MLF/lib/python3.8/site-packages/tensorflow_federated/python/learning/framework/dataset_reduce.py:28 _dataset_reduce_fn *\n return dataset.reduce(initial_state=initial_state_fn(), reduce_func=reduce_fn)\n /home/itec/bogdan/.conda/envs/MLF/lib/python3.8/site-packages/tensorflow_federated/python/learning/keras_utils.py:365 forward_pass *\n return self._forward_pass(batch_input, training=training)\n /home/itec/bogdan/.conda/envs/MLF/lib/python3.8/site-packages/tensorflow_federated/python/learning/keras_utils.py:322 _forward_pass *\n predictions = self._keras_model(inputs, training=training)\n /home/itec/bogdan/.conda/envs/MLF/lib/python3.8/site-packages/tensorflow/python/keras/engine/base_layer_v1.py:776 __call__ **\n outputs = call_fn(cast_inputs, *args, **kwargs)\n /home/itec/bogdan/.conda/envs/MLF/lib/python3.8/site-packages/tensorflow/python/keras/engine/sequential.py:372 call\n return super(Sequential, self).call(inputs, training=training, mask=mask)\n /home/itec/bogdan/.conda/envs/MLF/lib/python3.8/site-packages/tensorflow/python/keras/engine/functional.py:385 call\n return self._run_internal_graph(\n /home/itec/bogdan/.conda/envs/MLF/lib/python3.8/site-packages/tensorflow/python/keras/engine/functional.py:508 _run_internal_graph\n outputs = node.layer(*args, **kwargs)\n /home/itec/bogdan/.conda/envs/MLF/lib/python3.8/site-packages/tensorflow/python/keras/layers/recurrent.py:663 __call__\n return super(RNN, self).__call__(inputs, **kwargs)\n /home/itec/bogdan/.conda/envs/MLF/lib/python3.8/site-packages/tensorflow/python/keras/engine/base_layer_v1.py:751 __call__\n input_spec.assert_input_compatibility(self.input_spec, inputs,\n /home/itec/bogdan/.conda/envs/MLF/lib/python3.8/site-packages/tensorflow/python/keras/engine/input_spec.py:176 assert_input_compatibility\n raise ValueError('Input ' + str(input_index) + ' of layer ' +\n\n ValueError: Input 0 of layer lstm is incompatible with the layer: expected ndim=3, found ndim=2. Full shape received: [None, 50]\n"
]
}
],
"source": [
"def model_fn():\n",
" # We _must_ create a new model here, and _not_ capture it from an external\n",
" # scope. TFF will call this within different graph contexts.\n",
"\n",
" keras_model = getTextModel() #create_keras_model()\n",
" input_spec_aux = preprocessed_sample_dataset.element_spec\n",
" return tff.learning.from_keras_model(\n",
" keras_model,\n",
" input_spec= input_spec_aux,\n",
" loss=tf.keras.losses.SparseCategoricalCrossentropy(),\n",
" metrics=[tf.keras.metrics.SparseCategoricalAccuracy()])\n",
"\n",
"\n",
"iterative_process = tff.learning.build_federated_averaging_process(\n",
" model_fn,\n",
" client_optimizer_fn=lambda: tf.keras.optimizers.Adam(learning_rate=client_lr),\n",
" server_optimizer_fn=lambda: tf.keras.optimizers.SGD(learning_rate=server_lr))\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"\nPROCESSING BATCH\nTensor(\"args_0:0\", shape=(None,), dtype=string)\n\nPROCESSING BATCH\nTensor(\"args_0:0\", shape=(None,), dtype=string)\n\nPROCESSING BATCH\nTensor(\"args_0:0\", shape=(None,), dtype=string)\n\nPROCESSING BATCH\nTensor(\"args_0:0\", shape=(None,), dtype=string)\n\nPROCESSING BATCH\nTensor(\"args_0:0\", shape=(None,), dtype=string)\nNumber of client datasets: 4\nFirst dataset: <PrefetchDataset shapes: OrderedDict([(x, (None,)), (y, (None, 1))]), types: OrderedDict([(x, tf.string), (y, tf.int64)])>\n"
]
}
],
"source": [
"print(str(iterative_process.initialize.type_signature))\n",
"\n",
"state = iterative_process.initialize()\n",
"\n",
"tff_train_acc = []\n",
"tff_val_acc = []\n",
"tff_train_loss = []\n",
"tff_val_loss = []\n",
"\n",
"print(\"\\n\\n####EVALUATING MODEL\")"
]
}
]
}
\ No newline at end of file
src/participation-hub/federated-learning-microservice/app/processing/evaluation.py → src/participation-hub/federated-learning-microservice/app/processing/fake_news/evaluation.py
View file @ c53b87b9
import numpy as np
import tensorflow as tf
import tensorflow_federated as tff
from models import MnistModel
#from models import MnistModel
from models2 import model_fn
def evaluate(server_state):
keras_model = MnistModel()
keras_model = model_fn
keras_model.compile(
loss=tf.keras.losses.SparseCategoricalCrossentropy(),
metrics=[tf.keras.metrics.SparseCategoricalAccuracy()]
......
src/participation-hub/federated-learning-microservice/app/processing/federated_training_algorithm.py → src/participation-hub/federated-learning-microservice/app/processing/fake_news/federated_training_algorithm copy.py
View file @ c53b87b9
......@@ -2,6 +2,7 @@ import numpy as np
import tensorflow as tf
import tensorflow_federated as tff
from models import MnistModel
from models2 import model_fn
#tf_dataset_type = None
#model_weights_type = None
......
src/participation-hub/federated-learning-microservice/app/processing/fake_news/federated_training_algorithm.py 0 → 100644
View file @ c53b87b9
import numpy as np
import tensorflow as tf
import tensorflow_federated as tff
#from models import MnistModel
from models2 import model_fn
#tf_dataset_type = None
#model_weights_type = None
@tf.function
def client_update(model, dataset, server_weights, client_optimizer):
"""Performs training (using the server model weights) on the client's dataset."""
# Initialize the client model with the current server weights.
client_weights = model.trainable_variables
# Assign the server weights to the client model.
tf.nest.map_structure(lambda x, y: x.assign(y),
client_weights, server_weights)
# Use the client_optimizer to update the local model.
for batch in dataset:
with tf.GradientTape() as tape:
# Compute a forward pass on the batch of data
outputs = model.forward_pass(batch)
# Compute the corresponding gradient
grads = tape.gradient(outputs.loss, client_weights)
grads_and_vars = zip(grads, client_weights)
# Apply the gradient using a client optimizer.
client_optimizer.apply_gradients(grads_and_vars)
return client_weights
@tf.function
def server_update(model, mean_client_weights):
"""Updates the server model weights as the average of the client model weights."""
model_weights = model.trainable_variables
# Assign the mean client weights to the server model.
tf.nest.map_structure(lambda x, y: x.assign(y),
model_weights, mean_client_weights)
return model_weights
#Creating the initialization computation
@tff.tf_computation
def server_init():
model = model_fn()
return model.trainable_variables
@tff.federated_computation
def initialize_fn():
return tff.federated_value(server_init(), tff.SERVER)
whimsy_model = model_fn()
tf_dataset_type = tff.SequenceType(whimsy_model.input_spec)
model_weights_type = server_init.type_signature.result
@tff.tf_computation(tf_dataset_type, model_weights_type)
def client_update_fn(tf_dataset, server_weights):
model = model_fn()
client_optimizer = tf.keras.optimizers.SGD(learning_rate=0.01)
return client_update(model, tf_dataset, server_weights, client_optimizer)
@tff.tf_computation(model_weights_type)
def server_update_fn(mean_client_weights):
model = model_fn()
return server_update(model, mean_client_weights)
federated_server_type = tff.FederatedType(model_weights_type, tff.SERVER)
federated_dataset_type = tff.FederatedType(tf_dataset_type, tff.CLIENTS)
@tff.federated_computation(federated_server_type, federated_dataset_type)
def next_fn(server_weights, federated_dataset):
# Broadcast the server weights to the clients.
print("server_weights")
print(str(server_weights.type_signature))
server_weights_at_client = tff.federated_broadcast(server_weights)
# Each client computes their updated weights.
client_weights = tff.federated_map(
client_update_fn, (federated_dataset, server_weights_at_client))
# The server averages these updates.
mean_client_weights = tff.federated_mean(client_weights)
print("mean_client_wieghts")
print(str(mean_client_weights.type_signature))
# The server updates its model.
server_weights = tff.federated_map(server_update_fn, mean_client_weights)
return server_weights
def get_federated_algorithm():
#Creating the next_fn
#Getting the data type, needed explicitily in the modell functions
whimsy_model = model_fn()
global tf_dataset_type
tf_dataset_type = tff.SequenceType(whimsy_model.input_spec)
print("tf_dataset_type")
print(str(tf_dataset_type))
global model_weights_type
model_weights_type = server_init.type_signature.result
print("model_weights_type")
print(str(model_weights_type))
# finished printing types
federated_server_type = tff.FederatedType(model_weights_type, tff.SERVER)
federated_dataset_type = tff.FederatedType(tf_dataset_type, tff.CLIENTS)
federated_algorithm = tff.templates.IterativeProcess(
initialize_fn=initialize_fn,
next_fn=next_fn
)
return federated_algorithm
def merge_2_states(server1_state, server2_state):
return np.mean( np.array([ server1_state, server2_state ]), axis=0 )
\ No newline at end of file
src/participation-hub/federated-learning-microservice/app/processing/main_processing.py → src/participation-hub/federated-learning-microservice/app/processing/fake_news/main_processing.py
View file @ c53b87b9
......@@ -7,7 +7,7 @@ import tensorflow_federated as tff
from preprocessing import get_client_ids
from preprocessing import get_federated_train_data
from preprocessing import preprocess
from models import MnistModel
#from models import MnistModel
from federated_training_algorithm import get_federated_algorithm
from federated_training_algorithm import merge_2_states
from evaluation import evaluate
......@@ -44,7 +44,7 @@ for round in range(2):
server2_state = federated_algorithm.next(server2_state, federated_train_data)
merged_state = merge_2_states(server_state,server2_state)
print("server_state[1]")
print(server_state[1])
......
src/participation-hub/federated-learning-microservice/app/processing/fake_news/models2.py 0 → 100644
View file @ c53b87b9
import numpy as np
import tensorflow as tf
import tensorflow_federated as tff
import collections
def create_keras_model(): #### DEFAULT TEST MODEL
return tf.keras.models.Sequential([
tf.keras.layers.InputLayer(input_shape=(784,)),
tf.keras.layers.Dense(10, kernel_initializer='zeros'),
tf.keras.layers.Softmax(),
])
def model_fn():
# We _must_ create a new model here, and _not_ capture it from an external
# scope. TFF will call this within different graph contexts.
keras_model = create_keras_model() ###TODO CAN BE CHANGED TO INCLUDE CALLS TO OTHER MODELS (or it cannot, because you cannot specify which model you want to call?)
return tff.learning.from_keras_model(
keras_model,
input_spec=get_input_spec(),
loss=tf.keras.losses.SparseCategoricalCrossentropy(),
metrics=[tf.keras.metrics.SparseCategoricalAccuracy()])
def get_input_spec():
return collections.OrderedDict(
x=tf.TensorSpec([None, 784], tf.float32),
y=tf.TensorSpec([None, 1], tf.int32))
\ No newline at end of file
src/participation-hub/federated-learning-microservice/app/processing/fake_news/preprocessing copy.py 0 → 100644
View file @ c53b87b9
import collections
import attr
import functools
import numpy as np
import tensorflow as tf
import tensorflow_federated as tff
np.random.seed(0)
emnist_train, emnist_test = tff.simulation.datasets.emnist.load_data()
# NUM_CLIENTS = 10
# BATCH_SIZE = 20
def preprocess(dataset, BATCH_SIZE = 20):
def batch_format_fn(element):
"""Flatten a batch of EMNIST data and return a (features, label) tuple."""
return (tf.reshape(element['pixels'], [-1, 784]),
tf.reshape(element['label'], [-1, 1]))
return dataset.batch(BATCH_SIZE).map(batch_format_fn)
def get_client_ids(emnist_train, NUM_CLIENTS = 10):
return np.random.choice(emnist_train.client_ids, size=NUM_CLIENTS, replace=False)
#client_ids = np.random.choice(emnist_train.client_ids, size=NUM_CLIENTS, replace=False)
def get_federated_train_data(emnist_train, NUM_CLIENTS = 10):
client_ids = get_client_ids(emnist_train, NUM_CLIENTS)
return [preprocess(emnist_train.create_tf_dataset_for_client(x))
for x in client_ids
]
# federated_train_data = [preprocess(emnist_train.create_tf_dataset_for_client(x))
# for x in client_ids
# ]
##################################################################
## Second Dataset
############################################
def preprocess_and_shuffle(dataset):
"""Applies `preprocess_dataset` above and shuffles the result."""
preprocessed = preprocess(dataset)
return preprocessed.shuffle(buffer_size=5)
src/participation-hub/federated-learning-microservice/app/processing/fake_news/preprocessing.py 0 → 100644
View file @ c53b87b9
import collections
import attr
import functools
import numpy as np
import tensorflow as tf
import tensorflow_federated as tff
np.random.seed(0)
emnist_train, emnist_test = tff.simulation.datasets.emnist.load_data()
# NUM_CLIENTS = 10
# BATCH_SIZE = 20
def preprocess(dataset, BATCH_SIZE = 20):
def batch_format_fn(element):
"""Flatten a batch of EMNIST data and return a (features, label) tuple."""
return (tf.reshape(element['pixels'], [-1, 784]),
tf.reshape(element['label'], [-1, 1]))
return dataset.batch(BATCH_SIZE).map(batch_format_fn)
def get_client_ids(emnist_train, NUM_CLIENTS = 10):
return np.random.choice(emnist_train.client_ids, size=NUM_CLIENTS, replace=False)
#client_ids = np.random.choice(emnist_train.client_ids, size=NUM_CLIENTS, replace=False)
def get_federated_train_data(emnist_train, NUM_CLIENTS = 10):
client_ids = get_client_ids(emnist_train, NUM_CLIENTS)
return [preprocess(emnist_train.create_tf_dataset_for_client(x))
for x in client_ids
]
######
def preprocess_and_shuffle(dataset):
"""Applies `preprocess_dataset` above and shuffles the result."""
preprocessed = preprocess(dataset)
return preprocessed.shuffle(buffer_size=5)
#########################################################################################
#########################################################################################
#########################################################################################
src/participation-hub/federated-learning-microservice/app/processing/fake_news/prototype_db_fake_real/Fake.csv 0 → 100644
View file @ c53b87b9
This source diff could not be displayed because it is too large. You can view the blob instead.
src/participation-hub/federated-learning-microservice/app/processing/fake_news/prototype_db_fake_real/True.csv 0 → 100644
View file @ c53b87b9
This source diff could not be displayed because it is too large. You can view the blob instead.
src/participation-hub/federated-learning-microservice/app/processing/fake_news/prototype_federat_data copy.py 0 → 100644
View file @ c53b87b9
import pandas as pd
import re
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder
from tensorflow.keras.models import Model
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import LSTM, Activation, Dense, Dropout, Input, Embedding
from tensorflow.keras.optimizers import RMSprop
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing import sequence
from tensorflow.keras.callbacks import EarlyStopping
from tensorflow.keras.preprocessing.sequence import pad_sequences
real = pd.read_csv("processing/fake_news/prototype_db_fake_real/True.csv")
fake = pd.read_csv("processing/fake_news/prototype_db_fake_real/Fake.csv")
# dropping rows that have urls as text and date, real's dates look fine, also dropping ones that have no text
fake_drop = fake.drop(index=[9358,15507,15508,18933])
fake_drop = fake_drop.drop(fake_drop.loc[fake_drop.text == ' '].index)
real_drop = real.drop(real.loc[real.text == ' '].index)
# Give labels to data before combining
fake['fake'] = 1
real['fake'] = 0
combined = pd.concat([fake, real])
no_reuters = combined.copy()
no_reuters.text = no_reuters.text.str.replace('Reuters', '')
combined = no_reuters.copy()
## train/test split the text data and labels
features = combined['text']
labels = combined['fake']
X_train, X_test, y_train, y_test = train_test_split(features, labels, random_state = 42)
# the model will remember only the top 2000 most common words
max_words = 2000
max_len = 400
token = Tokenizer(num_words=max_words, lower=True, split=' ')
token.fit_on_texts(X_train.values)
sequences = token.texts_to_sequences(X_train.values)
train_sequences_padded = pad_sequences(sequences, maxlen=max_len)
embed_dim = 50
lstm_out = 64
batch_size = 32
input_shape_var = (max_words, )
model = Sequential()
model.add(Embedding(max_words, embed_dim, input_length = max_len, input_shape=input_shape_var))
model.add(LSTM(lstm_out))
model.add(Dense(256))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(1, name='out_layer'))
model.add(Activation('sigmoid'))
model.compile(loss = 'binary_crossentropy', optimizer='adam',\
metrics = ['accuracy'])
print(model.summary())
print()
history = model.fit(train_sequences_padded, y_train, batch_size=batch_size, epochs = 5, validation_split=0.2)
test_sequences = token.texts_to_sequences(X_test)
test_sequences_padded = pad_sequences(test_sequences, maxlen=max_len)
model.evaluate(test_sequences_padded, y_test)
###################################################################
experiment_name = "mnist"
method = "tff_training"
client_lr = 1e-2
server_lr = 1e-2
split = 4
NUM_ROUNDS = 5
NUM_EPOCHS = 5
BATCH_SIZE = 20
PREFETCH_BUFFER = 10
this_dir = Path.cwd()
model_dir = this_dir / "saved_models" / experiment_name / method
output_dir = this_dir / "results" / experiment_name / method
if not model_dir.exists():
model_dir.mkdir(parents=True)
if not output_dir.exists():
output_dir.mkdir(parents=True)
(x_train, y_train), (x_test, y_test) = mnist.load_data()
x_train = x_train.astype(np.float32)
y_train = y_train.astype(np.int32)
x_test = x_test.astype(np.float32).reshape(10000, 28, 28, 1)
y_test = y_test.astype(np.int32).reshape(10000, 1)
total_image_count = len(x_train)
image_per_set = int(np.floor(total_image_count/split))
client_train_dataset = collections.OrderedDict()
for i in range(1, split+1):
client_name = "client_" + str(i)
start = image_per_set * (i-1)
end = image_per_set * i
print(f"Adding data from {start} to {end} for client : {client_name}")
data = collections.OrderedDict((('label', y_train[start:end]), ('pixels', x_train[start:end])))
client_train_dataset[client_name] = data
train_dataset = tff.simulation.FromTensorSlicesClientData(client_train_dataset)
sample_dataset = train_dataset.create_tf_dataset_for_client(train_dataset.client_ids[0])
sample_element = next(iter(sample_dataset))
SHUFFLE_BUFFER = image_per_set
\ No newline at end of file
src/participation-hub/federated-learning-microservice/app/processing/fake_news/prototype_federat_data.py 0 → 100644
View file @ c53b87b9
import pandas as pd
import re
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder
from tensorflow.keras.models import Model
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import LSTM, Activation, Dense, Dropout, Input, Embedding
from tensorflow.keras.optimizers import RMSprop
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing import sequence
from tensorflow.keras.callbacks import EarlyStopping
from tensorflow.keras.preprocessing.sequence import pad_sequences
from nltk import word_tokenize
import collections
import numpy as np
import tensorflow as tf
import tensorflow_federated as tff
real = pd.read_csv("processing/fake_news/prototype_db_fake_real/True.csv")
fake = pd.read_csv("processing/fake_news/prototype_db_fake_real/Fake.csv")
# dropping rows that have urls as text and date, real's dates look fine, also dropping ones that have no text
fake_drop = fake.drop(index=[9358,15507,15508,18933])
fake_drop = fake_drop.drop(fake_drop.loc[fake_drop.text == ' '].index)
real_drop = real.drop(real.loc[real.text == ' '].index)
# Give labels to data before combining
fake['fake'] = 1
real['fake'] = 0
combined = pd.concat([fake, real])
no_reuters = combined.copy()
no_reuters.text = no_reuters.text.str.replace('Reuters', '')
combined = no_reuters.copy()
## train/test split the text data and labels
features = combined['text'] #features is now
labels = combined['fake']
print ("SHAPE OF FEATURES")
print (features.shape)
print("TYPE")
print (type(features[0]))
print(" features 0")
print (features[0])
try:
features = features.apply(word_tokenize)
except Exception as e:
print (e)
print ("SHAPE OF FEATURES")
print (features.shape)
print("TYPE")
print (type(features[0]))
print(" features 0")
print (features[0])
X_train, X_test, y_train, y_test = train_test_split(features, labels, random_state = 42)
print (type(X_train))
NUM_CLIENTS = 4
#######
#TODO
lenght_of_traindataset = len(X_train) #uita-te jos cum e facut
items_per_set = int(np.floor(lenght_of_traindataset/NUM_CLIENTS))
client_train_dataset = collections.OrderedDict()
for i in range(1, NUM_CLIENTS+1):
client_name = "client_" + str(i)
start = items_per_set * (i-1)
end = items_per_set * i
print(f"Adding data from {start} to {end} for client : {client_name}")
data = collections.OrderedDict((('label', y_train[start:end]), ('features', X_train[start:end]))) #features in this case is equivalent to 'text', in image_classifiers case, it's the 'pixels'
client_train_dataset[client_name] = data
#########
####
#token = Tokenizer(num_words=max_words, lower=True, split=' ')
#token.fit_on_texts(X_train.values)
#sequences = token.texts_to_sequences(X_train.values)
#train_sequences_padded = pad_sequences(sequences, maxlen=max_len)
# def getTextModel2():
# model = Sequential()
# model.add(Embedding(max_words, embed_dim, input_length = max_len, input_shape=input_shape_var))
# return model
####################################################################
# def getTextModel():
# model = Sequential()
# model.add(Embedding(max_words, embed_dim, input_length = max_len, input_shape=input_shape_var))
# model.add(LSTM(lstm_out))
# model.add(Dense(256))
# model.add(Activation('relu'))
# model.add(Dropout(0.5))
# model.add(Dense(1, name='out_layer'))
# model.add(Activation('sigmoid'))
# #model.compile(loss = 'binary_crossentropy', optimizer='adam',\
# #metrics = ['accuracy'])
# return model
# embed_dim = 50
# lstm_out = 64
# batch_size = 32
# input_shape_var = (max_words, )
# model = getTextModel()
# print(model.summary())
# print()
#TRAINING?
#history = model.fit(train_sequences_padded, y_train, batch_size=batch_size, epochs = 5, validation_split=0.2)
#test_sequences = token.texts_to_sequences(X_test)
#test_sequences_padded = pad_sequences(test_sequences, maxlen=max_len)
#model.evaluate(test_sequences_padded, y_test)
###################################################################
"""
experiment_name = "mnist"
method = "tff_training"
client_lr = 1e-2
server_lr = 1e-2
NUM_CLIENTS = 4
NUM_ROUNDS = 5
NUM_EPOCHS = 5
BATCH_SIZE = 20
PREFETCH_BUFFER = 10
this_dir = Path.cwd()
model_dir = this_dir / "saved_models" / experiment_name / method
output_dir = this_dir / "results" / experiment_name / method
if not model_dir.exists():
model_dir.mkdir(parents=True)
if not output_dir.exists():
output_dir.mkdir(parents=True)
(x_train, y_train), (x_test, y_test) = mnist.load_data()
x_train = x_train.astype(np.float32)
y_train = y_train.astype(np.int32)
x_test = x_test.astype(np.float32).reshape(10000, 28, 28, 1)
y_test = y_test.astype(np.int32).reshape(10000, 1)
total_image_count = len(x_train)
items_per_set = int(np.floor(total_image_count/NUM_CLIENTS))
client_train_dataset = collections.OrderedDict()
for i in range(1, NUM_CLIENTS+1):
client_name = "client_" + str(i)
start = items_per_set * (i-1)
end = items_per_set * i
print(f"Adding data from {start} to {end} for client : {client_name}")
data = collections.OrderedDict((('label', y_train[start:end]), ('pixels', x_train[start:end])))
client_train_dataset[client_name] = data
"""
#client_train_dataset = np.asarray(client_train_dataset).astype('string')
train_dataset = tff.simulation.FromTensorSlicesClientData(client_train_dataset)
sample_dataset = train_dataset.create_tf_dataset_for_client(train_dataset.client_ids[0])
sample_element = next(iter(sample_dataset))
SHUFFLE_BUFFER = items_per_set
#################TESTAT PANA AICI ^
method = "tff_training"
client_lr = 1e-2
server_lr = 1e-2
NUM_CLIENTS = 4
NUM_ROUNDS = 5
NUM_EPOCHS = 5
BATCH_SIZE = 20
PREFETCH_BUFFER = 10
def preprocess(dataset):
def batch_format_fn(element):
"""Flatten a batch `pixels` and return the features as an `OrderedDict`."""
#return element
print("")
print("PROCESSING BATCH")
print(element['features'])
return collections.OrderedDict(
x=element['features'],#tf.reshape(element['pixels'], [-1, 28, 28, 1]),
y=tf.reshape(element['label'], [-1, 1])
)
return dataset.repeat(NUM_EPOCHS).shuffle(SHUFFLE_BUFFER).batch(
BATCH_SIZE).map(batch_format_fn).prefetch(PREFETCH_BUFFER)
preprocessed_sample_dataset = preprocess(sample_dataset)
#usless sample_batch = nest.map_structure(lambda x: x.numpy(), next(iter(preprocessed_sample_dataset)))
def make_federated_data(client_data, client_ids):
return [preprocess(client_data.create_tf_dataset_for_client(x)) for x in client_ids]
federated_train_data = make_federated_data(train_dataset, train_dataset.client_ids)
print('Number of client datasets: {l}'.format(l=len(federated_train_data)))
print('First dataset: {d}'.format(d=federated_train_data[0]))
auxvar = preprocessed_sample_dataset.element_spec
# the model will remember only the top 2000 most common words
max_words = 2000
max_len = 400
embed_dim = 50
lstm_out = 64
batch_size = 32
#input_shape_var = (max_words, )
def getTextModel():
model = Sequential()
model.add(Embedding(max_words, embed_dim, input_length = max_len))#, input_shape=auxvar))
model.add(LSTM(lstm_out))
model.add(Dense(256))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(1, name='out_layer'))
model.add(Activation('sigmoid'))
#model.compile(loss = 'binary_crossentropy', optimizer='adam',\
#metrics = ['accuracy'])
return model
model = getTextModel()
print(model.summary())
def model_fn():
# We _must_ create a new model here, and _not_ capture it from an external
# scope. TFF will call this within different graph contexts.
keras_model = getTextModel() #create_keras_model()
input_spec_aux = preprocessed_sample_dataset.element_spec
return tff.learning.from_keras_model(
keras_model,
input_spec= input_spec_aux,
loss=tf.keras.losses.SparseCategoricalCrossentropy(),
metrics=[tf.keras.metrics.SparseCategoricalAccuracy()])
iterative_process = tff.learning.build_federated_averaging_process(
model_fn,
client_optimizer_fn=lambda: tf.keras.optimizers.Adam(learning_rate=client_lr),
server_optimizer_fn=lambda: tf.keras.optimizers.SGD(learning_rate=server_lr))
print(str(iterative_process.initialize.type_signature))
state = iterative_process.initialize()
tff_train_acc = []
tff_val_acc = []
tff_train_loss = []
tff_val_loss = []
print("\n\n####EVALUATING MODEL")
# eval_model = None
# for round_num in range(1, NUM_ROUNDS+1):
# state, tff_metrics = iterative_process.next(state, federated_train_data)
# eval_model = getTextModel()#create_keras_model()
# eval_model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=client_lr),
# loss=tf.keras.losses.SparseCategoricalCrossentropy(),
# metrics=[tf.keras.metrics.SparseCategoricalAccuracy()])
# tff.learning.assign_weights_to_keras_model(eval_model, state.model)
# ev_result = eval_model.evaluate(x_test, y_test, verbose=0)
# print('round {:2d}, metrics={}'.format(round_num, tff_metrics))
# print(f"Eval loss : {ev_result[0]} and Eval accuracy : {ev_result[1]}")
# tff_train_acc.append(float(tff_metrics.sparse_categorical_accuracy))
# tff_val_acc.append(ev_result[1])
# tff_train_loss.append(float(tff_metrics.loss))
# tff_val_loss.append(ev_result[0])
# metric_collection = {"sparse_categorical_accuracy": tff_train_acc,
# "val_sparse_categorical_accuracy": tff_val_acc,
# "loss": tff_train_loss,
# "val_loss": tff_val_loss}
# if eval_model:
# eval_model.save(model_dir / (experiment_name + ".h5"))
# else:
# print("training didn't started")
# exit()
# fig = plt.figure(figsize=(10, 6))
# plot_graph(list(range(1, 26))[4::5], tff_train_acc, label='Train Accuracy')
# plot_graph(list(range(1, 26))[4::5], tff_val_acc, label='Validation Accuracy')
# plt.legend()
# plt.savefig(output_dir / "federated_model_Accuracy.png")
# plt.figure(figsize=(10, 6))
# plot_graph(list(range(1, 26))[4::5], tff_train_loss, label='Train loss')
# plot_graph(list(range(1, 26))[4::5], tff_val_loss, label='Validation loss')
# plt.legend()
# plt.savefig(output_dir / "federated_model_loss.png")
# # saving metric values to text file
# txt_file_path = output_dir / (experiment_name + ".txt")
# with open(txt_file_path.as_posix(), "w") as handle:
# content = []
# for key, val in metric_collection.items():
# line_content = key
# val = [str(k) for k in val]
# line_content = line_content + " " + " ".join(val)
# content.append(line_content)
# handle.write("\n".join(content))
\ No newline at end of file
src/participation-hub/federated-learning-microservice/app/processing/fake_news/prototype_fkns.py 0 → 100644
View file @ c53b87b9
# import numpy as np ##
# import pandas as pd ##
# import seaborn as sns
# import matplotlib.pyplot as plt
# import nltk ##
# from sklearn.preprocessing import LabelBinarizer
# from nltk.corpus import stopwords ##
# from nltk.stem.porter import PorterStemmer #
# from wordcloud import WordCloud,STOPWORDS ##
# from nltk.stem import WordNetLemmatizer #
# from nltk.tokenize import word_tokenize,sent_tokenize #
# from bs4 import BeautifulSoup ##
# import re,string,unicodedata ##
# from keras.preprocessing import text, sequence
# from sklearn.metrics import classification_report,confusion_matrix,accuracy_score
# from sklearn.model_selection import train_test_split
# from string import punctuation ##
# from nltk import pos_tag #
# from nltk.corpus import wordnet #
# import keras
# from keras.models import Sequential
# from keras.layers import Dense,Embedding,LSTM,Dropout
# from keras.callbacks import ReduceLROnPlateau
# import tensorflow as tf
# import tensorflow_federated as tff
import pandas as pd
import re
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder
from tensorflow.keras.models import Model
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import LSTM, Activation, Dense, Dropout, Input, Embedding
from tensorflow.keras.optimizers import RMSprop
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing import sequence
from tensorflow.keras.callbacks import EarlyStopping
from tensorflow.keras.preprocessing.sequence import pad_sequences
# def get_dataframe_OLD():
# true = pd.read_csv("processing/fake_news/prototype_db_fake_real/True.csv")
# false = pd.read_csv("processing/fake_news/prototype_db_fake_real/Fake.csv")
# true['category'] = 1
# false['category'] = 0
# df = pd.concat([true,false]) #Merging the 2 datasets
# df['text'] = df['text'] + " " + df['title']
# del df['title']
# del df['subject']
# del df['date']
# return df
# def strip_html(text):
# soup = BeautifulSoup(text, "html.parser")
# return soup.get_text()
# #Removing the square brackets
# def remove_between_square_brackets(text):
# return re.sub('\[[^]]*\]', '', text)
# # Removing URL's
# def remove_between_square_brackets(text):
# return re.sub(r'http\S+', '', text)
# #Removing the stopwords from text
# def remove_stopwords(text):
# stop = set(nltk.corpus.stopwords.words('english'))
# punctuation = list(string.punctuation)
# stop.update(punctuation)
# final_text = []
# for i in text.split():
# if i.strip().lower() not in stop:
# final_text.append(i.strip())
# return " ".join(final_text)
# #Removing the noisy text
# def denoise_text(text):
# text = strip_html(text)
# text = remove_between_square_brackets(text)
# text = remove_stopwords(text)
# return text
# #Apply function on review column
# def data_cleaning():
# df = get_dataframe()
# df['text']=df['text'].apply(denoise_text)
# print("#####")
#data_cleaning()
# x_train,x_test,y_train,y_test = train_test_split(df.text,df.category,random_state = 0)
# max_features = 10000
# maxlen = 300
# tokenizer = text.Tokenizer(num_words=max_features)
# tokenizer.fit_on_texts(x_train)
# tokenized_train = tokenizer.texts_to_sequences(x_train)
# x_train = sequence.pad_sequences(tokenized_train, maxlen=maxlen)
# #Defining Neural Network
# model = Sequential()
# #Non-trainable embeddidng layer
# model.add(Embedding(max_features, output_dim=embed_size, weights=[embedding_matrix], input_length=maxlen, trainable=False))
# #LSTM
# model.add(LSTM(units=128 , return_sequences = True , recurrent_dropout = 0.25 , dropout = 0.25))
# model.add(LSTM(units=64 , recurrent_dropout = 0.1 , dropout = 0.1))
# model.add(Dense(units = 32 , activation = 'relu'))
# model.add(Dense(1, activation='sigmoid'))
# model.compile(optimizer=keras.optimizers.Adam(lr = 0.01), loss='binary_crossentropy', metrics=['accuracy'])
real = pd.read_csv("processing/fake_news/prototype_db_fake_real/True.csv")
fake = pd.read_csv("processing/fake_news/prototype_db_fake_real/Fake.csv")
# dropping rows that have urls as text and date, real's dates look fine, also dropping ones that have no text
fake_drop = fake.drop(index=[9358,15507,15508,18933])
fake_drop = fake_drop.drop(fake_drop.loc[fake_drop.text == ' '].index)
real_drop = real.drop(real.loc[real.text == ' '].index)
# Give labels to data before combining
fake['fake'] = 1
real['fake'] = 0
combined = pd.concat([fake, real])
no_reuters = combined.copy()
no_reuters.text = no_reuters.text.str.replace('Reuters', '')
combined = no_reuters.copy()
## train/test split the text data and labels
features = combined['text']
labels = combined['fake']
X_train, X_test, y_train, y_test = train_test_split(features, labels, random_state = 42)
# the model will remember only the top 2000 most common words
max_words = 2000
max_len = 400
token = Tokenizer(num_words=max_words, lower=True, split=' ')
token.fit_on_texts(X_train.values)
sequences = token.texts_to_sequences(X_train.values)
train_sequences_padded = pad_sequences(sequences, maxlen=max_len)
embed_dim = 50
lstm_out = 64
batch_size = 32
input_shape_var = (max_words, )
model = Sequential()
model.add(Embedding(max_words, embed_dim, input_length = max_len, input_shape=input_shape_var))
model.add(LSTM(lstm_out))
model.add(Dense(256))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(1, name='out_layer'))
model.add(Activation('sigmoid'))
model.compile(loss = 'binary_crossentropy', optimizer='adam',\
metrics = ['accuracy'])
print(model.summary())
print()
history = model.fit(train_sequences_padded, y_train, batch_size=batch_size, epochs = 5, validation_split=0.2)
test_sequences = token.texts_to_sequences(X_test)
test_sequences_padded = pad_sequences(test_sequences, maxlen=max_len)
model.evaluate(test_sequences_padded, y_test)
\ No newline at end of file
src/participation-hub/federated-learning-microservice/app/processing/prototype/evaluation.py 0 → 100644
View file @ c53b87b9
import numpy as np
import tensorflow as tf
import tensorflow_federated as tff
#from models import MnistModel
from models2 import model_fn
def evaluate(server_state):
keras_model = model_fn
keras_model.compile(
loss=tf.keras.losses.SparseCategoricalCrossentropy(),
metrics=[tf.keras.metrics.SparseCategoricalAccuracy()]
)
keras_model.set_weights(server_state)
keras_model.evaluate(central_emnist_test)
def evaluate2(server_state,tff_learning_model):
#TODO: Assign weights to the model
#First idea = server_update function??
evaluation = tff.learning.build_federated_evaluation(tff_learning_model)
# keras_model = MnistModel()
# keras_model.compile(
# loss=tf.keras.losses.SparseCategoricalCrossentropy(),
# metrics=[tf.keras.metrics.SparseCategoricalAccuracy()]
# )
# keras_model.set_weights(server_state)
# keras_model.evaluate(central_emnist_test)
src/participation-hub/federated-learning-microservice/app/processing/prototype/federated_training_algorithm copy.py 0 → 100644
View file @ c53b87b9
import numpy as np
import tensorflow as tf
import tensorflow_federated as tff
from models import MnistModel
from models2 import model_fn
#tf_dataset_type = None
#model_weights_type = None
@tf.function
def client_update(model, dataset, server_weights, client_optimizer):
"""Performs training (using the server model weights) on the client's dataset."""
# Initialize the client model with the current server weights.
client_weights = model.trainable_variables
# Assign the server weights to the client model.
tf.nest.map_structure(lambda x, y: x.assign(y),
client_weights, server_weights)
# Use the client_optimizer to update the local model.
for batch in dataset:
with tf.GradientTape() as tape:
# Compute a forward pass on the batch of data
outputs = model.forward_pass(batch)
# Compute the corresponding gradient
grads = tape.gradient(outputs.loss, client_weights)
grads_and_vars = zip(grads, client_weights)
# Apply the gradient using a client optimizer.
client_optimizer.apply_gradients(grads_and_vars)
return client_weights
@tf.function
def server_update(model, mean_client_weights):
"""Updates the server model weights as the average of the client model weights."""
model_weights = model.trainable_variables
# Assign the mean client weights to the server model.
tf.nest.map_structure(lambda x, y: x.assign(y),
model_weights, mean_client_weights)
return model_weights
#Creating the initialization computation
@tff.tf_computation
def server_init():
model = MnistModel() #model_fn()
return model.trainable_variables
@tff.federated_computation
def initialize_fn():
return tff.federated_value(server_init(), tff.SERVER)
whimsy_model = MnistModel()
tf_dataset_type = tff.SequenceType(whimsy_model.input_spec)
model_weights_type = server_init.type_signature.result
@tff.tf_computation(tf_dataset_type, model_weights_type)
def client_update_fn(tf_dataset, server_weights):
model = MnistModel()#model_fn()
client_optimizer = tf.keras.optimizers.SGD(learning_rate=0.01)
return client_update(model, tf_dataset, server_weights, client_optimizer)
@tff.tf_computation(model_weights_type)
def server_update_fn(mean_client_weights):
model = MnistModel()#model_fn()
return server_update(model, mean_client_weights)
federated_server_type = tff.FederatedType(model_weights_type, tff.SERVER)
federated_dataset_type = tff.FederatedType(tf_dataset_type, tff.CLIENTS)
@tff.federated_computation(federated_server_type, federated_dataset_type)
def next_fn(server_weights, federated_dataset):
# Broadcast the server weights to the clients.
print("server_weights")
print(str(server_weights.type_signature))
server_weights_at_client = tff.federated_broadcast(server_weights)
# Each client computes their updated weights.
client_weights = tff.federated_map(
client_update_fn, (federated_dataset, server_weights_at_client))
# The server averages these updates.
mean_client_weights = tff.federated_mean(client_weights)
print("mean_client_wieghts")
print(str(mean_client_weights.type_signature))
# The server updates its model.
server_weights = tff.federated_map(server_update_fn, mean_client_weights)
return server_weights
def get_federated_algorithm():
#Creating the next_fn
#Getting the data type, needed explicitily in the modell functions
whimsy_model = MnistModel() #model_fn()
global tf_dataset_type
tf_dataset_type = tff.SequenceType(whimsy_model.input_spec)
print("tf_dataset_type")
print(str(tf_dataset_type))
global model_weights_type
model_weights_type = server_init.type_signature.result
print("model_weights_type")
print(str(model_weights_type))
# finished printing types
federated_server_type = tff.FederatedType(model_weights_type, tff.SERVER)
federated_dataset_type = tff.FederatedType(tf_dataset_type, tff.CLIENTS)
federated_algorithm = tff.templates.IterativeProcess(
initialize_fn=initialize_fn,
next_fn=next_fn
)
return federated_algorithm
def merge_2_states(server_state1, server_state2):
return np.mean( np.array([ server_state, server2_state ]), axis=0 )
\ No newline at end of file
src/participation-hub/federated-learning-microservice/app/processing/prototype/federated_training_algorithm.py 0 → 100644
View file @ c53b87b9
import numpy as np
import tensorflow as tf
import tensorflow_federated as tff
#from models import MnistModel
from models2 import model_fn
#tf_dataset_type = None
#model_weights_type = None
@tf.function
def client_update(model, dataset, server_weights, client_optimizer):
"""Performs training (using the server model weights) on the client's dataset."""
# Initialize the client model with the current server weights.
client_weights = model.trainable_variables
# Assign the server weights to the client model.
tf.nest.map_structure(lambda x, y: x.assign(y),
client_weights, server_weights)
# Use the client_optimizer to update the local model.
for batch in dataset:
with tf.GradientTape() as tape:
# Compute a forward pass on the batch of data
outputs = model.forward_pass(batch)
# Compute the corresponding gradient
grads = tape.gradient(outputs.loss, client_weights)
grads_and_vars = zip(grads, client_weights)
# Apply the gradient using a client optimizer.
client_optimizer.apply_gradients(grads_and_vars)
return client_weights
@tf.function
def server_update(model, mean_client_weights):
"""Updates the server model weights as the average of the client model weights."""
model_weights = model.trainable_variables
# Assign the mean client weights to the server model.
tf.nest.map_structure(lambda x, y: x.assign(y),
model_weights, mean_client_weights)
return model_weights
#Creating the initialization computation
@tff.tf_computation
def server_init():
model = model_fn()
return model.trainable_variables
@tff.federated_computation
def initialize_fn():
return tff.federated_value(server_init(), tff.SERVER)
whimsy_model = model_fn()
tf_dataset_type = tff.SequenceType(whimsy_model.input_spec)
model_weights_type = server_init.type_signature.result
@tff.tf_computation(tf_dataset_type, model_weights_type)
def client_update_fn(tf_dataset, server_weights):
model = model_fn()
client_optimizer = tf.keras.optimizers.SGD(learning_rate=0.01)
return client_update(model, tf_dataset, server_weights, client_optimizer)
@tff.tf_computation(model_weights_type)
def server_update_fn(mean_client_weights):
model = model_fn()
return server_update(model, mean_client_weights)
federated_server_type = tff.FederatedType(model_weights_type, tff.SERVER)
federated_dataset_type = tff.FederatedType(tf_dataset_type, tff.CLIENTS)
@tff.federated_computation(federated_server_type, federated_dataset_type)
def next_fn(server_weights, federated_dataset):
# Broadcast the server weights to the clients.
print("server_weights")
print(str(server_weights.type_signature))
server_weights_at_client = tff.federated_broadcast(server_weights)
# Each client computes their updated weights.
client_weights = tff.federated_map(
client_update_fn, (federated_dataset, server_weights_at_client))
# The server averages these updates.
mean_client_weights = tff.federated_mean(client_weights)
print("mean_client_wieghts")
print(str(mean_client_weights.type_signature))
# The server updates its model.
server_weights = tff.federated_map(server_update_fn, mean_client_weights)
return server_weights
def get_federated_algorithm():
#Creating the next_fn
#Getting the data type, needed explicitily in the modell functions
whimsy_model = model_fn()
global tf_dataset_type
tf_dataset_type = tff.SequenceType(whimsy_model.input_spec)
print("tf_dataset_type")
print(str(tf_dataset_type))
global model_weights_type
model_weights_type = server_init.type_signature.result
print("model_weights_type")
print(str(model_weights_type))
# finished printing types
federated_server_type = tff.FederatedType(model_weights_type, tff.SERVER)
federated_dataset_type = tff.FederatedType(tf_dataset_type, tff.CLIENTS)
federated_algorithm = tff.templates.IterativeProcess(
initialize_fn=initialize_fn,
next_fn=next_fn
)
return federated_algorithm
def merge_2_states(server1_state, server2_state):
return np.mean( np.array([ server1_state, server2_state ]), axis=0 )
\ No newline at end of file
src/participation-hub/federated-learning-microservice/app/processing/prototype/main_processing.py 0 → 100644
View file @ c53b87b9
import collections
import attr
import functools
import numpy as np
import tensorflow as tf
import tensorflow_federated as tff
from preprocessing import get_client_ids
from preprocessing import get_federated_train_data
from preprocessing import preprocess
#from models import MnistModel
from federated_training_algorithm import get_federated_algorithm
from federated_training_algorithm import merge_2_states
from evaluation import evaluate
np.random.seed(0)
print("## Starting...")
#GET THE DATA (for now it's the default dataset)
emnist_train, emnist_test = tff.simulation.datasets.emnist.load_data()
######client_ids = get_client_ids(emnist_train) not used
print("## Preprocessing the federated_train_data")
federated_train_data = get_federated_train_data(emnist_train)
print("## Declaring the model")
#it is done in models.py
print("## Declaring the federated algorithm")
federated_algorithm = get_federated_algorithm()
server_state = federated_algorithm.initialize()
for round in range(20):
server_state = federated_algorithm.next(server_state, federated_train_data)
print("server_state type")
print(str(type(server_state)))
print(str(type(server_state[0])))
print("FINISHEEED")
server2_state = federated_algorithm.initialize()
for round in range(2):
server2_state = federated_algorithm.next(server2_state, federated_train_data)
merged_state = merge_2_states(server_state,server2_state)
print("server_state[1]")
print(server_state[1])
print("server2_state[1]")
print(server2_state[1])
print("merged_state[1]")
print(merged_state[1])
# print("federated_algorithm.initialize.type_signature")
# print(str(federated_algorithm.initialize.type_signature)
# print("federated_algorithm.next.type_signature")
# print(str(federated_algorithm.next.type_signature))
# print("## Training the model")
# iterative_process = tff.learning.build_federated_averaging_process(
# MnistModel,
# client_optimizer_fn=lambda: tf.keras.optimizers.SGD(learning_rate=0.02))
# state = iterative_process.initialize()
# for round_num in range(1, 11):
# state, metrics = iterative_process.next(state, federated_train_data)
# print('round {:2d}, metrics={}'.format(round_num, metrics))
#evaluation = tff.learning.build_federated_evaluation(MnistModel)
#TODO integration
print("## Evaluation of the model")
\ No newline at end of file
src/participation-hub/federated-learning-microservice/app/processing/prototype/model_mnist_OLD.py 0 → 100644
View file @ c53b87b9
import numpy as np
import tensorflow as tf
import tensorflow_federated as tff
import collections
input_spec_data_global = None
MnistVariables = collections.namedtuple(
'MnistVariables', 'weights bias num_examples loss_sum accuracy_sum')
def create_mnist_variables():
return MnistVariables(
weights=tf.Variable(
lambda: tf.zeros(dtype=tf.float32, shape=(784, 10)),
name='weights',
trainable=True),
bias=tf.Variable(
lambda: tf.zeros(dtype=tf.float32, shape=(10)),
name='bias',
trainable=True),
num_examples=tf.Variable(0.0, name='num_examples', trainable=False),
loss_sum=tf.Variable(0.0, name='loss_sum', trainable=False),
accuracy_sum=tf.Variable(0.0, name='accuracy_sum', trainable=False))
def mnist_forward_pass(variables, batch):
y = tf.nn.softmax(tf.matmul(batch['x'], variables.weights) + variables.bias)
predictions = tf.cast(tf.argmax(y, 1), tf.int32)
flat_labels = tf.reshape(batch['y'], [-1])
loss = -tf.reduce_mean(
tf.reduce_sum(tf.one_hot(flat_labels, 10) * tf.math.log(y), axis=[1]))
accuracy = tf.reduce_mean(
tf.cast(tf.equal(predictions, flat_labels), tf.float32))
num_examples = tf.cast(tf.size(batch['y']), tf.float32)
variables.num_examples.assign_add(num_examples)
variables.loss_sum.assign_add(loss * num_examples)
variables.accuracy_sum.assign_add(accuracy * num_examples)
return loss, predictions
def get_local_mnist_metrics(variables):
return collections.OrderedDict(
num_examples=variables.num_examples,
loss=variables.loss_sum / variables.num_examples,
accuracy=variables.accuracy_sum / variables.num_examples)
@tff.federated_computation
def aggregate_mnist_metrics_across_clients(metrics):
return collections.OrderedDict(
num_examples=tff.federated_sum(metrics.num_examples),
loss=tff.federated_mean(metrics.loss, metrics.num_examples),
accuracy=tff.federated_mean(metrics.accuracy, metrics.num_examples))
class MnistModel(tff.learning.Model):
def __init__(self):
self._variables = create_mnist_variables()
@property
def trainable_variables(self):
return [self._variables.weights, self._variables.bias]
@property
def non_trainable_variables(self):
return []
@property
def local_variables(self):
return [
self._variables.num_examples, self._variables.loss_sum,
self._variables.accuracy_sum
]
@property
def input_spec(self):
return collections.OrderedDict(
x=tf.TensorSpec([None, 784], tf.float32),
y=tf.TensorSpec([None, 1], tf.int32))
@tf.function
def forward_pass(self, batch, training=True):
del training
loss, predictions = mnist_forward_pass(self._variables, batch)
num_exmaples = tf.shape(batch['x'])[0]
return tff.learning.BatchOutput(
loss=loss, predictions=predictions, num_examples=num_exmaples)
@tf.function
def report_local_outputs(self):
return get_local_mnist_metrics(self._variables)
@property
def federated_output_computation(self):
return aggregate_mnist_metrics_across_clients
\ No newline at end of file
src/participation-hub/federated-learning-microservice/app/processing/prototype/models2.py 0 → 100644
View file @ c53b87b9
import numpy as np
import tensorflow as tf
import tensorflow_federated as tff
import collections
def create_keras_model(): #### DEFAULT TEST MODEL
return tf.keras.models.Sequential([
tf.keras.layers.InputLayer(input_shape=(784,)),
tf.keras.layers.Dense(10, kernel_initializer='zeros'),
tf.keras.layers.Softmax(),
])
def model_fn():
# We _must_ create a new model here, and _not_ capture it from an external
# scope. TFF will call this within different graph contexts.
keras_model = create_keras_model() ###TODO CAN BE CHANGED TO INCLUDE CALLS TO OTHER MODELS (or it cannot, because you cannot specify which model you want to call?)
return tff.learning.from_keras_model(
keras_model,
input_spec=get_input_spec(),
loss=tf.keras.losses.SparseCategoricalCrossentropy(),
metrics=[tf.keras.metrics.SparseCategoricalAccuracy()])
def get_input_spec():
return collections.OrderedDict(
x=tf.TensorSpec([None, 784], tf.float32),
y=tf.TensorSpec([None, 1], tf.int32))
\ No newline at end of file
src/participation-hub/federated-learning-microservice/app/processing/text_processing/global_hyperparams.py 0 → 100644
View file @ c53b87b9
def initialize():
global MAX_LENGTH #Lenght of sentences to be fed into the NN. Similar to image size i.e. 100pixels x 100pixels, but it's 1D.
MAX_LENGTH = 40
global VOCAB_SIZE #tells how many words it memorises. each word is stored as an int.
VOCAB_SIZE = 6000
global NUM_CLIENTS #number of clients in the federated dataset
NUM_CLIENTS = 4
global SHUFFLE_BUFFER
SHUFFLE_BUFFER = 5000 #used in preprocessing
global BATCH_SIZE
BATCH_SIZE = 512 #size of the batch of the dataset. which is later fed into the NN.
global INPUT_SPEC #tell the model how the data will look like.
INPUT_SPEC = None #must & will be initialized after data preprocessing. Currently it's being initialised with: train_dataset[0].element_spec
global EMBED_DIM # number of dimension of the embedding of the layer in the model.
EMBED_DIM = 10
global EPOCHS #number of epochs the model will be trained
EPOCHS = 5
\ No newline at end of file
src/participation-hub/federated-learning-microservice/app/processing/text_processing/main_proc.py 0 → 100644
View file @ c53b87b9
import processing.text_processing.global_hyperparams as globals
if __name__ == "__main__":
#globals.initialize()
src/participation-hub/federated-learning-microservice/app/processing/text_processing/preprocessing.py 0 → 100644
View file @ c53b87b9
import pandas as pd
from sklearn.model_selection import train_test_split
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import LSTM, Dense, Dropout, Embedding
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences
import collections
import numpy as np
import tensorflow as tf
import tensorflow_federated as tff
real = pd.read_csv("processing/fake_news/prototype_db_fake_real/True.csv")
fake = pd.read_csv("processing/fake_news/prototype_db_fake_real/Fake.csv")
# dropping rows that have urls as text and date, real's dates look fine, also dropping ones that have no text
fake_drop = fake.drop(index=[9358,15507,15508,18933])
fake_drop = fake_drop.drop(fake_drop.loc[fake_drop.text == ' '].index)
real_drop = real.drop(real.loc[real.text == ' '].index)
# Give labels to data before combining
fake['label'] = 1
real['label'] = 0
combined = pd.concat([fake, real])
no_reuters = combined.copy()
no_reuters.text = no_reuters.text.str.replace('Reuters', '')
combined = no_reuters.copy()
## train/test split the text data and labels
df_text = combined['text'] #features is now
labels = combined['label'] #or maybe use target?
target = combined['label'].values
tokenizer = Tokenizer(oov_token = "<OOV>", num_words=6000)
tokenizer.fit_on_texts(df_text)
MAX_LENGTH = 40
VOCAB_SIZE = 6000
sequences_train = tokenizer.texts_to_sequences(df_text)
padded_train = pad_sequences(sequences_train, padding = 'post', maxlen=MAX_LENGTH)
#Data_train, data_text, label_train, label_test
X_train, X_test, y_train, y_test = train_test_split(padded_train, target, test_size=0.2)
X_train = tf.convert_to_tensor(X_train)
X_test = tf.convert_to_tensor(X_test)
y_train = tf.convert_to_tensor(y_train)
y_test = tf.convert_to_tensor(y_test)
src/participation-hub/federated-learning-microservice/app/processing/text_processing/prototype_fed_text.py 0 → 100644
View file @ c53b87b9
# import pandas as pd
# train_df = pd.read_csv('processing/fake-news/train.csv', header=0)
# test_df = pd.read_csv('processing/fake-news/test.csv', header=0)
# train_df = train_df.fillna(' ')
# test_df = test_df.fillna(' ')
# train_df['all_info'] = train_df['text'] + train_df['title'] + train_df['author']
# test_df['all_info'] = test_df['text'] + test_df['title'] + test_df['author']
# target = train_df['label'].values
import pandas as pd
from sklearn.model_selection import train_test_split
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import LSTM, Dense, Dropout, Embedding
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences
import collections
import numpy as np
import tensorflow as tf
import tensorflow_federated as tff
real = pd.read_csv("processing/fake_news/prototype_db_fake_real/True.csv")
fake = pd.read_csv("processing/fake_news/prototype_db_fake_real/Fake.csv")
# dropping rows that have urls as text and date, real's dates look fine, also dropping ones that have no text
fake_drop = fake.drop(index=[9358,15507,15508,18933])
fake_drop = fake_drop.drop(fake_drop.loc[fake_drop.text == ' '].index)
real_drop = real.drop(real.loc[real.text == ' '].index)
# Give labels to data before combining
fake['label'] = 1
real['label'] = 0
combined = pd.concat([fake, real])
no_reuters = combined.copy()
no_reuters.text = no_reuters.text.str.replace('Reuters', '')
combined = no_reuters.copy()
## train/test split the text data and labels
df_text = combined['text'] #features is now
labels = combined['label'] #or maybe use target?
target = combined['label'].values
print("##################label")
print(type(labels))
print(labels)
print("###########")
print(type(combined['label'].values))
print(combined['label'].values)
print("df_text_type:")
print(type(df_text))
############################ ^ORIGINAL DB
# train_df = pd.read_csv('processing/text_processing/prototype_db_fake_real/train.csv', header=0)
# test_df = pd.read_csv('processing/text_processing/prototype_db_fake_real/test.csv', header=0)
# train_df = train_df.fillna(' ')
# test_df = test_df.fillna(' ')
# train_df['all_info'] = train_df['text'] + train_df['title'] + train_df['author']
# test_df['all_info'] = test_df['text'] + test_df['title'] + test_df['author']
# target = train_df['label'].values
# print(type(train_df['label'].values))
# print(train_df['label'].values)
# df_text = train_df['all_info']
######################################################################################
tokenizer = Tokenizer(oov_token = "<OOV>", num_words=6000)
tokenizer.fit_on_texts(df_text)
MAX_LENGTH = 40
VOCAB_SIZE = 6000
sequences_train = tokenizer.texts_to_sequences(df_text)
padded_train = pad_sequences(sequences_train, padding = 'post', maxlen=MAX_LENGTH)
#Data_train, data_text, label_train, label_test
X_train, X_test, y_train, y_test = train_test_split(padded_train, target, test_size=0.2)
X_train = tf.convert_to_tensor(X_train)
X_test = tf.convert_to_tensor(X_test)
y_train = tf.convert_to_tensor(y_train)
y_test = tf.convert_to_tensor(y_test)
print(X_train.shape)
print(y_train.shape)
print("Type of X_train, X_test, y_train, y_test")
print(type(X_train))
print(type(X_test))
print(type(y_train))
print(type(y_test))
###################################################################################\
#FED PREPROCESSING
NUM_CLIENTS = 4
SHUFFLE_BUFFER = 5000
BATCH_SIZE = 512
def preprocess(dataset):
def element_fn(x, y):
return collections.OrderedDict([
('x', x),
('y', y)#tf.cast(tf.reshape(y, [1]), tf.float32))
])
return dataset.map(element_fn).shuffle(
SHUFFLE_BUFFER).batch(BATCH_SIZE)
def generate_clients_datasets(n, source_x, source_y):
clients_dataset=[]
for i in range(n):
dataset=tf.data.Dataset.from_tensor_slices(([source_x[i]], [source_y[i]]))
dataset=preprocess(dataset)
clients_dataset.append(dataset)
return clients_dataset
train_dataset=generate_clients_datasets(NUM_CLIENTS, X_train, y_train)
test_dataset=generate_clients_datasets(NUM_CLIENTS, X_test, y_test)
# Grab a single batch of data so that TFF knows what data looks like.
# sample_batch = tf.nest.map_structure(
# lambda x: x.numpy(), iter(train_dataset[0]).next())
INPUT_SPEC = train_dataset[0].element_spec
print("DONE PREPROCESSING")
#################################################################################
EMBED_DIM = 10
def get_simple_LSTM_model():
model = Sequential()
model.add(Embedding(VOCAB_SIZE, EMBED_DIM, input_length=MAX_LENGTH))
model.add(Dropout(0.3))
model.add(LSTM(100))
model.add(Dropout(0.3))
model.add(Dense(64, activation='relu'))
model.add(Dropout(0.3))
model.add(Dense(1, activation='sigmoid'))
# model.compile(loss='binary_crossentropy',
# optimizer='adam',
# metrics=[tf.keras.metrics.Precision(), tf.keras.metrics.Recall()])
return model
def model_fn():
keras_model = get_simple_LSTM_model()
#return tff.learning.from_compiled_keras_model(keras_model, sample_batch) original
return tff.learning.from_keras_model(
keras_model,
input_spec=INPUT_SPEC,
loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
metrics=[tf.keras.metrics.SparseCategoricalAccuracy()])
# Training and evaluating the model
iterative_process = tff.learning.build_federated_averaging_process(model_fn,client_optimizer_fn=lambda: tf.keras.optimizers.SGD(lr=0.5))
state = iterative_process.initialize()
EPOCHS = 5
for n in range(EPOCHS):
state, metrics = iterative_process.next(state, train_dataset)
print('round {}, training metrics={}'.format(n+1, metrics))
evaluation = tff.learning.build_federated_evaluation(model_fn)
eval_metrics = evaluation(state.model, train_dataset)
print('Training evaluation metrics={}'.format(eval_metrics))
test_metrics = evaluation(state.model, test_dataset)
print('Test evaluation metrics={}'.format(test_metrics))
# model = get_simple_LSTM_model()
# print(model.summary())
# best_model_file_name = "processing/text_processing/models/best_model_LSTM.hdf5"
# callbacks=[
# tf.keras.callbacks.EarlyStopping(monitor="val_loss", patience=15,
# verbose=1, mode="min", restore_best_weights=True),
# tf.keras.callbacks.ModelCheckpoint(filepath=best_model_file_name, verbose=1, save_best_only=True)
# ]
# history = model.fit(X_train,
# y_train,
# epochs=EPOCHS,
# validation_data=(X_test, y_test),
# callbacks=callbacks)
# model.save(best_model_file_name)
# model = tf.keras.models.load_model(best_model_file_name)
src/participation-hub/federated-learning-microservice/app/processing/text_processing/prototype_text_LSTM_nonfed.py 0 → 100644
View file @ c53b87b9
# import pandas as pd
# train_df = pd.read_csv('processing/fake-news/train.csv', header=0)
# test_df = pd.read_csv('processing/fake-news/test.csv', header=0)
# train_df = train_df.fillna(' ')
# test_df = test_df.fillna(' ')
# train_df['all_info'] = train_df['text'] + train_df['title'] + train_df['author']
# test_df['all_info'] = test_df['text'] + test_df['title'] + test_df['author']
# target = train_df['label'].values
import pandas as pd
import re
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder
from tensorflow.keras.models import Model
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import LSTM, Activation, Dense, Dropout, Input, Embedding
from tensorflow.keras.optimizers import RMSprop
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing import sequence
from tensorflow.keras.callbacks import EarlyStopping
from tensorflow.keras.preprocessing.sequence import pad_sequences
from nltk import word_tokenize
import collections
import numpy as np
import tensorflow as tf
import tensorflow_federated as tff
real = pd.read_csv("processing/fake_news/prototype_db_fake_real/True.csv")
fake = pd.read_csv("processing/fake_news/prototype_db_fake_real/Fake.csv")
# dropping rows that have urls as text and date, real's dates look fine, also dropping ones that have no text
fake_drop = fake.drop(index=[9358,15507,15508,18933])
fake_drop = fake_drop.drop(fake_drop.loc[fake_drop.text == ' '].index)
real_drop = real.drop(real.loc[real.text == ' '].index)
# Give labels to data before combining
fake['label'] = 1
real['label'] = 0
combined = pd.concat([fake, real])
no_reuters = combined.copy()
no_reuters.text = no_reuters.text.str.replace('Reuters', '')
combined = no_reuters.copy()
## train/test split the text data and labels
train_df_text = combined['text'] #features is now
labels = combined['label'] #or maybe use target?
target = combined['label'].values
print("##################label")
print(type(labels))
print(labels)
print("###########")
print(type(combined['label'].values))
print(combined['label'].values)
print("train_df_type:")
print(type(train_df_text))
############################ ^ORIGINAL DB
# train_df = pd.read_csv('processing/text_processing/prototype_db_fake_real/train.csv', header=0)
# test_df = pd.read_csv('processing/text_processing/prototype_db_fake_real/test.csv', header=0)
# train_df = train_df.fillna(' ')
# test_df = test_df.fillna(' ')
# train_df['all_info'] = train_df['text'] + train_df['title'] + train_df['author']
# test_df['all_info'] = test_df['text'] + test_df['title'] + test_df['author']
# target = train_df['label'].values
# print(type(train_df['label'].values))
# print(train_df['label'].values)
# train_df_text = train_df['all_info']
######################################################################################
tokenizer = Tokenizer(oov_token = "<OOV>", num_words=6000)
tokenizer.fit_on_texts(train_df_text)
max_length = 40
vocab_size = 6000
sequences_train = tokenizer.texts_to_sequences(train_df_text)
padded_train = pad_sequences(sequences_train, padding = 'post', maxlen=max_length)
X_train, X_test, y_train, y_test = train_test_split(padded_train, target, test_size=0.2)
print(X_train.shape)
print(y_train.shape)
#################################################################################
embed_dim = 10
def get_simple_LSTM_model():
model = Sequential()
model.add(Embedding(vocab_size, embed_dim, input_length=max_length))
model.add(Dropout(0.3))
model.add(LSTM(100))
model.add(Dropout(0.3))
model.add(Dense(64, activation='relu'))
model.add(Dropout(0.3))
model.add(Dense(1, activation='sigmoid'))
return model
model = get_simple_LSTM_model()
print(model.summary())
best_model_file_name = "processing/text_processing/models/best_model_LSTM.hdf5"
callbacks=[
tf.keras.callbacks.EarlyStopping(monitor="val_loss", patience=15,
verbose=1, mode="min", restore_best_weights=True),
tf.keras.callbacks.ModelCheckpoint(filepath=best_model_file_name, verbose=1, save_best_only=True)
]
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=[tf.keras.metrics.Precision(), tf.keras.metrics.Recall()])
history = model.fit(X_train,
y_train,
epochs=5,
validation_data=(X_test, y_test),
callbacks=callbacks)
model.save(best_model_file_name)
model = tf.keras.models.load_model(best_model_file_name)
src/participation-hub/federated-learning-microservice/app/slurm-1780.out 0 → 100644
View file @ c53b87b9
2021-05-05 12:34:15.886296: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
Traceback (most recent call last):
File "processing/main_processing.py", line 11, in <module>
from federated_training_algorithm import get_federated_algorithm
File "/home/itec/bogdan/Articonf/smart/src/participation-hub/federated-learning-microservice/app/processing/federated_training_algorithm.py", line 48, in <module>
def server_init():
File "/home/itec/bogdan/.conda/envs/MLF/lib/python3.8/site-packages/tensorflow_federated/python/core/impl/wrappers/computation_wrapper.py", line 407, in __call__
result = fn_to_wrap(*args, **kwargs)
File "/home/itec/bogdan/Articonf/smart/src/participation-hub/federated-learning-microservice/app/processing/federated_training_algorithm.py", line 49, in server_init
model = model_fn()
File "/home/itec/bogdan/Articonf/smart/src/participation-hub/federated-learning-microservice/app/processing/models2.py", line 16, in model_fn
return tff.learning.from_keras_model(
NameError: name 'tff' is not defined
src/participation-hub/federated-learning-microservice/app/slurm-1781.out 0 → 100644
View file @ c53b87b9
2021-05-05 12:36:34.308397: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
Traceback (most recent call last):
File "processing/main_processing.py", line 11, in <module>
from federated_training_algorithm import get_federated_algorithm
File "/home/itec/bogdan/Articonf/smart/src/participation-hub/federated-learning-microservice/app/processing/federated_training_algorithm.py", line 48, in <module>
def server_init():
File "/home/itec/bogdan/.conda/envs/MLF/lib/python3.8/site-packages/tensorflow_federated/python/core/impl/wrappers/computation_wrapper.py", line 407, in __call__
result = fn_to_wrap(*args, **kwargs)
File "/home/itec/bogdan/Articonf/smart/src/participation-hub/federated-learning-microservice/app/processing/federated_training_algorithm.py", line 49, in server_init
model = model_fn()
File "/home/itec/bogdan/Articonf/smart/src/participation-hub/federated-learning-microservice/app/processing/models2.py", line 19, in model_fn
input_spec=preprocessed_example_dataset.element_spec,
NameError: name 'preprocessed_example_dataset' is not defined
src/participation-hub/federated-learning-microservice/app/slurm-1782.out 0 → 100644
View file @ c53b87b9
2021-05-05 12:43:03.812706: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
Traceback (most recent call last):
File "processing/main_processing.py", line 11, in <module>
from federated_training_algorithm import get_federated_algorithm
File "/home/itec/bogdan/Articonf/smart/src/participation-hub/federated-learning-microservice/app/processing/federated_training_algorithm.py", line 48, in <module>
def server_init():
File "/home/itec/bogdan/.conda/envs/MLF/lib/python3.8/site-packages/tensorflow_federated/python/core/impl/wrappers/computation_wrapper.py", line 407, in __call__
result = fn_to_wrap(*args, **kwargs)
File "/home/itec/bogdan/Articonf/smart/src/participation-hub/federated-learning-microservice/app/processing/federated_training_algorithm.py", line 49, in server_init
model = model_fn()
File "/home/itec/bogdan/Articonf/smart/src/participation-hub/federated-learning-microservice/app/processing/models2.py", line 19, in model_fn
input_spec=get_input_spec(),
File "/home/itec/bogdan/Articonf/smart/src/participation-hub/federated-learning-microservice/app/processing/models2.py", line 25, in get_input_spec
return collections.OrderedDict(
NameError: name 'collections' is not defined
src/participation-hub/federated-learning-microservice/app/slurm-1783.out 0 → 100644
View file @ c53b87b9
2021-05-05 12:44:48.174832: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
2021-05-05 12:45:01.547785: W tensorflow/stream_executor/platform/default/dso_loader.cc:59] Could not load dynamic library 'libcuda.so.1'; dlerror: libcuda.so.1: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64
2021-05-05 12:45:01.548090: W tensorflow/stream_executor/cuda/cuda_driver.cc:312] failed call to cuInit: UNKNOWN ERROR (303)
2021-05-05 12:45:01.548116: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:156] kernel driver does not appear to be running on this host (mcore2): /proc/driver/nvidia/version does not exist
2021-05-05 12:45:01.624528: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN)to use the following CPU instructions in performance-critical operations: AVX2 FMA
To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
2021-05-05 12:45:01.674208: I tensorflow/core/platform/profile_utils/cpu_utils.cc:104] CPU Frequency: 2599705000 Hz
2021-05-05 12:45:01.676283: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x55727649d8f0 initialized for platform Host (this does not guarantee that XLA will be used). Devices:
2021-05-05 12:45:01.676306: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version
Traceback (most recent call last):
File "processing/main_processing.py", line 56, in <module>
print(merged_state[1])
NameError: name 'merged_state' is not defined
src/participation-hub/federated-learning-microservice/app/slurm-1784.out 0 → 100644
View file @ c53b87b9
2021-05-05 12:47:00.299027: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
2021-05-05 12:47:14.153090: W tensorflow/stream_executor/platform/default/dso_loader.cc:59] Could not load dynamic library 'libcuda.so.1'; dlerror: libcuda.so.1: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64
2021-05-05 12:47:14.153503: W tensorflow/stream_executor/cuda/cuda_driver.cc:312] failed call to cuInit: UNKNOWN ERROR (303)
2021-05-05 12:47:14.153524: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:156] kernel driver does not appear to be running on this host (mcore2): /proc/driver/nvidia/version does not exist
2021-05-05 12:47:14.220064: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN)to use the following CPU instructions in performance-critical operations: AVX2 FMA
To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
2021-05-05 12:47:14.250921: I tensorflow/core/platform/profile_utils/cpu_utils.cc:104] CPU Frequency: 2599705000 Hz
2021-05-05 12:47:14.252596: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x55caa388f420 initialized for platform Host (this does not guarantee that XLA will be used). Devices:
2021-05-05 12:47:14.252622: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version
src/participation-hub/federated-learning-microservice/app/slurm-1785.out 0 → 100644
View file @ c53b87b9
2021-05-05 12:48:43.541178: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
2021-05-05 12:48:57.548367: W tensorflow/stream_executor/platform/default/dso_loader.cc:59] Could not load dynamic library 'libcuda.so.1'; dlerror: libcuda.so.1: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64
2021-05-05 12:48:57.548773: W tensorflow/stream_executor/cuda/cuda_driver.cc:312] failed call to cuInit: UNKNOWN ERROR (303)
2021-05-05 12:48:57.548823: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:156] kernel driver does not appear to be running on this host (mcore2): /proc/driver/nvidia/version does not exist
2021-05-05 12:48:57.617636: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN)to use the following CPU instructions in performance-critical operations: AVX2 FMA
To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
2021-05-05 12:48:57.658912: I tensorflow/core/platform/profile_utils/cpu_utils.cc:104] CPU Frequency: 2599705000 Hz
2021-05-05 12:48:57.661020: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x55e17f536420 initialized for platform Host (this does not guarantee that XLA will be used). Devices:
2021-05-05 12:48:57.661049: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version
src/participation-hub/federated-learning-microservice/app/slurm-1786.out 0 → 100644
View file @ c53b87b9
2021-05-05 12:53:08.876262: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
2021-05-05 12:53:22.787924: W tensorflow/stream_executor/platform/default/dso_loader.cc:59] Could not load dynamic library 'libcuda.so.1'; dlerror: libcuda.so.1: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64
2021-05-05 12:53:22.788250: W tensorflow/stream_executor/cuda/cuda_driver.cc:312] failed call to cuInit: UNKNOWN ERROR (303)
2021-05-05 12:53:22.788293: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:156] kernel driver does not appear to be running on this host (mcore2): /proc/driver/nvidia/version does not exist
Traceback (most recent call last):
File "processing/main_processing.py", line 11, in <module>
from federated_training_algorithm import get_federated_algorithm
File "/home/itec/bogdan/Articonf/smart/src/participation-hub/federated-learning-microservice/app/processing/federated_training_algorithm.py", line 56, in <module>
whimsy_model = MnistModel()
NameError: name 'MnistModel' is not defined
src/participation-hub/federated-learning-microservice/app/slurm-1787.out 0 → 100644
View file @ c53b87b9
2021-05-05 12:55:42.203681: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
2021-05-05 12:55:56.142028: W tensorflow/stream_executor/platform/default/dso_loader.cc:59] Could not load dynamic library 'libcuda.so.1'; dlerror: libcuda.so.1: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64
2021-05-05 12:55:56.142493: W tensorflow/stream_executor/cuda/cuda_driver.cc:312] failed call to cuInit: UNKNOWN ERROR (303)
2021-05-05 12:55:56.142516: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:156] kernel driver does not appear to be running on this host (mcore2): /proc/driver/nvidia/version does not exist
2021-05-05 12:55:56.215449: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN)to use the following CPU instructions in performance-critical operations: AVX2 FMA
To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
2021-05-05 12:55:56.225527: I tensorflow/core/platform/profile_utils/cpu_utils.cc:104] CPU Frequency: 2599705000 Hz
2021-05-05 12:55:56.227527: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x55be268bd420 initialized for platform Host (this does not guarantee that XLA will be used). Devices:
2021-05-05 12:55:56.227548: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version
src/participation-hub/federated-learning-microservice/app/slurm-1788.out 0 → 100644
View file @ c53b87b9
2021-05-05 13:51:03.378007: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
Traceback (most recent call last):
File "processing/main_processing.py", line 10, in <module>
from models import MnistModel
ModuleNotFoundError: No module named 'models'
src/participation-hub/federated-learning-microservice/app/slurm-1789.out 0 → 100644
View file @ c53b87b9
2021-05-05 14:16:11.554269: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
2021-05-05 14:16:28.615094: W tensorflow/stream_executor/platform/default/dso_loader.cc:59] Could not load dynamic library 'libcuda.so.1'; dlerror: libcuda.so.1: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64
2021-05-05 14:16:28.615308: W tensorflow/stream_executor/cuda/cuda_driver.cc:312] failed call to cuInit: UNKNOWN ERROR (303)
2021-05-05 14:16:28.615342: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:156] kernel driver does not appear to be running on this host (mcore2): /proc/driver/nvidia/version does not exist
2021-05-05 14:16:28.696253: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN)to use the following CPU instructions in performance-critical operations: AVX2 FMA
To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
2021-05-05 14:16:28.725734: I tensorflow/core/platform/profile_utils/cpu_utils.cc:104] CPU Frequency: 2599705000 Hz
2021-05-05 14:16:28.727680: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x56288f42e0d0 initialized for platform Host (this does not guarantee that XLA will be used). Devices:
2021-05-05 14:16:28.727704: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version
Traceback (most recent call last):
File "processing/main_processing.py", line 13, in <module>
from evaluation import evaluate
File "/home/itec/bogdan/Articonf/smart/src/participation-hub/federated-learning-microservice/app/processing/evaluation.py", line 4, in <module>
from models import MnistModel
ModuleNotFoundError: No module named 'models'
src/participation-hub/federated-learning-microservice/app/slurm-1790.out 0 → 100644
View file @ c53b87b9
2021-05-05 14:19:11.932075: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
2021-05-05 14:19:25.830410: W tensorflow/stream_executor/platform/default/dso_loader.cc:59] Could not load dynamic library 'libcuda.so.1'; dlerror: libcuda.so.1: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64
2021-05-05 14:19:25.830831: W tensorflow/stream_executor/cuda/cuda_driver.cc:312] failed call to cuInit: UNKNOWN ERROR (303)
2021-05-05 14:19:25.830869: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:156] kernel driver does not appear to be running on this host (mcore2): /proc/driver/nvidia/version does not exist
2021-05-05 14:19:25.902017: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN)to use the following CPU instructions in performance-critical operations: AVX2 FMA
To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
2021-05-05 14:19:25.911762: I tensorflow/core/platform/profile_utils/cpu_utils.cc:104] CPU Frequency: 2599705000 Hz
2021-05-05 14:19:25.913382: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x563c6edbf450 initialized for platform Host (this does not guarantee that XLA will be used). Devices:
2021-05-05 14:19:25.913401: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version
src/participation-hub/federated-learning-microservice/app/slurm-1791.out 0 → 100644
View file @ c53b87b9
python: can't open file 'processing/main_processing.py': [Errno 2] No such file or directory
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment