Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
S
SMART
Project
Project
Details
Activity
Releases
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
3
Issues
3
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Registry
Registry
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
UNI-KLU
SMART
Commits
fec9c13c
Commit
fec9c13c
authored
Jun 15, 2021
by
Bogdan Mihai (ARTICONF student)
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Deleted obsolete DB files
parent
e1216230
Changes
25
Show whitespace changes
Inline
Side-by-side
Showing
25 changed files
with
97 additions
and
735 deletions
+97
-735
routes.yml
...ub/federated-learning-microservice/app/configs/routes.yml
+9
-59
layer_adapter.py
...ed-learning-microservice/app/db/entities/layer_adapter.py
+0
-109
table.py
.../federated-learning-microservice/app/db/entities/table.py
+0
-26
use_case.py
...derated-learning-microservice/app/db/entities/use_case.py
+0
-37
repository.py
...-hub/federated-learning-microservice/app/db/repository.py
+0
-62
table_repository.py
...ederated-learning-microservice/app/db/table_repository.py
+0
-47
use_case_repository.py
...rated-learning-microservice/app/db/use_case_repository.py
+0
-51
main.py
...icipation-hub/federated-learning-microservice/app/main.py
+0
-2
__init__.py
...-microservice/app/processing/crowd_journalism/__init__.py
+0
-0
checkpoint_manager.py
...ice/app/processing/crowd_journalism/checkpoint_manager.py
+0
-0
federated_algorithm.py
...ce/app/processing/crowd_journalism/federated_algorithm.py
+82
-0
main_proc.py
...microservice/app/processing/crowd_journalism/main_proc.py
+0
-0
ledger.csv
...ng-microservice/app/processing/text_processing/ledger.csv
+2
-0
saved_model.pb
...ing/text_processing/models/ckpt_1623768325/saved_model.pb
+0
-0
variables.data-00000-of-00001
...s/ckpt_1623768325/variables/variables.data-00000-of-00001
+0
-0
variables.index
...ocessing/models/ckpt_1623768325/variables/variables.index
+0
-0
saved_model.pb
...ing/text_processing/models/ckpt_1623768510/saved_model.pb
+0
-0
variables.data-00000-of-00001
...s/ckpt_1623768510/variables/variables.data-00000-of-00001
+0
-0
variables.index
...ocessing/models/ckpt_1623768510/variables/variables.index
+0
-0
developers.py
.../federated-learning-microservice/app/routes/developers.py
+1
-1
layer.py
...n-hub/federated-learning-microservice/app/routes/layer.py
+0
-228
owners.py
...-hub/federated-learning-microservice/app/routes/owners.py
+3
-2
tables.py
...-hub/federated-learning-microservice/app/routes/tables.py
+0
-60
use_case.py
...ub/federated-learning-microservice/app/routes/use_case.py
+0
-38
test_layer_adapter.py
...ted-learning-microservice/app/tests/test_layer_adapter.py
+0
-13
No files found.
src/participation-hub/federated-learning-microservice/app/configs/routes.yml
View file @
fec9c13c
swagger
:
'
2.0'
paths
:
#####
# USE-CASES
#####
/use-cases
:
post
:
security
:
-
JwtRegular
:
[]
operationId
:
"
routes.use_case.add"
tags
:
-
"
Use-Cases"
summary
:
"
Adds
a
new
Use-Case"
description
:
"
Adds
a
new
Use-Case"
parameters
:
-
in
:
body
name
:
"
Object"
required
:
true
schema
:
type
:
object
properties
:
name
:
type
:
string
example
:
"
use_case_1"
description
:
"
Unique
name
for
the
Use-Case"
responses
:
'
200'
:
description
:
"
Successful
Request"
'
400'
:
description
:
"
Use-Case
already
exists"
get
:
security
:
-
JwtRegular
:
[]
operationId
:
"
routes.use_case.all"
tags
:
-
"
Use-Cases"
summary
:
"
Retrieves
all
Use-Cases"
description
:
"
Retrieves
all
Use-Cases"
responses
:
'
200'
:
description
:
"
Successful
Request"
delete
:
security
:
-
JwtAdmin
:
[]
operationId
:
"
routes.use_case.delete_all"
tags
:
-
"
Use-Cases"
summary
:
"
Delete
all
Use-Cases"
description
:
"
Delete
all
Use-Cases"
responses
:
'
200'
:
description
:
"
Successful
Request"
#####
# Owners
#####
/Owners/use_case/{use_case}/last_train
:
...
...
@@ -228,12 +178,12 @@ paths:
definitions
:
Data_entry
:
type
:
"
object"
required
:
-
internal
properties
:
internal
:
type
:
string
example
:
"
todo,
create
the
preprocessing
part"
\ No newline at end of file
# definitions:
# Data_entry:
# type: "object"
# required:
# - internal
# properties:
# internal:
# type: string
# example: "todo, create the preprocessing part"
\ No newline at end of file
src/participation-hub/federated-learning-microservice/app/db/entities/layer_adapter.py
deleted
100644 → 0
View file @
e1216230
from
typing
import
Dict
,
List
class
LayerAdapter
:
'''
represents the mapping from an internal layer with a set of attributes and what
attributes from the dataset correspond to each one
'''
def
__init__
(
self
,
name
:
str
,
use_case
:
str
,
table
:
str
,
properties
:
List
[
str
],
cluster_properties
:
List
[
str
]):
'''
Creates a new instance of LayerAdapter
@params:
name - Required : unique identifier for the layer
properties - Required : list of mappings from the Schema
cluster_properties - Required : subset of properties to indicate what properties are used for clustering
use_case - Required : identifier for the use-case this layer belongs to
'''
self
.
name
=
name
self
.
properties
=
properties
self
.
use_case
=
use_case
self
.
table
=
table
for
prop
in
cluster_properties
:
if
prop
not
in
properties
:
raise
ValueError
(
f
"{prop} is no property in the layer!"
)
self
.
cluster_properties
=
cluster_properties
def
add_mapping
(
self
,
internal
:
str
):
'''
Add a new mapping to the layer. This mapping consists of an internal representation.
@params:
internal - Required: string identifier used internally, f.e. "startTime"
external - Required: string identifier the column has in the external dataset f.e. "arrivalTimeOfCustomer"
'''
if
internal
not
in
self
.
properties
:
self
.
properties
.
append
(
internal
)
def
delete_mapping
(
self
,
internal
:
str
):
'''
Removes a proprety from the layer. Raises a ValueError if the property identified by the internal representation
does not exist.
@params:
internal - Required: string identifier used internally, f.e. "startTime"
'''
if
internal
not
in
self
.
properties
:
raise
ValueError
(
f
"Attribute {internal} is not an internal attribute!"
)
if
internal
in
self
.
cluster_properties
:
self
.
delete_cluster_mapping
(
internal
)
self
.
properties
.
remove
(
internal
)
def
add_cluster_mapping
(
self
,
attribute
:
str
):
'''
Adds an attribute of the internal representations to the set of attributes to cluster after. The set of cluster
attribute is always a subset of self.properties.keys().
@params:
attribute - Required: string identifier used internally, f.e. "startTime"
'''
if
attribute
not
in
self
.
properties
:
raise
ValueError
(
f
"Attribute {attribute} is not an internal attribute!"
)
if
attribute
not
in
self
.
cluster_properties
:
self
.
cluster_properties
.
append
(
attribute
)
def
delete_cluster_mapping
(
self
,
attribute
:
str
):
'''
Removes an attribute from the set of cluster-attributes. Raises a ValueError if either the attribute does not exist
as an internal representation
@params:
attribute - Required: string identifier used internally, f.e. "startTime"
'''
if
attribute
not
in
self
.
properties
:
raise
ValueError
(
f
"Attribute {attribute} is not an internal attribute!"
)
self
.
cluster_properties
.
remove
(
attribute
)
def
to_serializable_dict
(
self
)
->
Dict
:
return
{
"name"
:
self
.
name
,
"properties"
:
self
.
properties
,
"cluster_properties"
:
self
.
cluster_properties
,
"use_case"
:
self
.
use_case
,
"table"
:
self
.
table
}
@
staticmethod
def
from_serializable_dict
(
user_dict
:
Dict
):
'''
creates a layer object from a dictionary. has to have the following keys:
- name
- properties
- cluster_properties
- use_case
'''
return
LayerAdapter
(
user_dict
[
"name"
],
user_dict
[
"use_case"
],
user_dict
[
"table"
],
user_dict
[
"properties"
],
user_dict
[
"cluster_properties"
]
)
src/participation-hub/federated-learning-microservice/app/db/entities/table.py
deleted
100644 → 0
View file @
e1216230
from
typing
import
Dict
class
Table
:
def
__init__
(
self
,
use_case
:
str
,
name
:
str
,
mappings
:
Dict
[
str
,
str
]):
self
.
use_case
=
use_case
self
.
name
=
name
self
.
mappings
=
mappings
def
to_serializable_dict
(
self
)
->
Dict
:
return
{
"name"
:
self
.
name
,
"use_case"
:
self
.
use_case
,
"mappings"
:
self
.
mappings
}
def
add_mapping
(
self
,
internal
:
str
,
external
:
str
):
if
internal
not
in
self
.
mappings
.
keys
():
self
.
mappings
[
internal
]
=
external
@
staticmethod
def
from_serializable_dict
(
data
:
Dict
):
return
Table
(
data
[
"use_case"
],
data
[
"name"
],
data
[
"mappings"
]
)
src/participation-hub/federated-learning-microservice/app/db/entities/use_case.py
deleted
100644 → 0
View file @
e1216230
from
typing
import
Dict
,
List
class
UseCase
:
'''
represents the mapping from an internal layer with a set of attributes and what
attributes from the dataset correspond to each one
'''
def
__init__
(
self
,
name
:
str
):
'''
Creates a new instance of LayerAdapter
@params:
name - Required : unique identifier for the layer
properties - Required : maps InternalPropertyName->DatasetPropertyName
cluster_properties - Required : subset of the keys of properties, marks properties to cluster with
use_case - Required : identifier for the use-case this layer belongs to
'''
self
.
name
=
name
def
to_serializable_dict
(
self
)
->
Dict
:
return
{
"name"
:
self
.
name
,
}
@
staticmethod
def
from_serializable_dict
(
user_dict
:
Dict
):
'''
creates a layer object from a dictionary. has to have the following keys:
- name
- properties
- cluster_properties
- use_case
'''
return
UseCase
(
user_dict
[
"name"
],
)
src/participation-hub/federated-learning-microservice/app/db/repository.py
deleted
100644 → 0
View file @
e1216230
# global imports (dont't worry, red is normal)
import
network_constants
as
netconst
from
database.MongoRepositoryBase
import
MongoRepositoryBase
from
db.entities.layer_adapter
import
LayerAdapter
from
db.entities.use_case
import
UseCase
import
pymongo
import
json
from
typing
import
List
,
Dict
class
Repository
(
MongoRepositoryBase
):
'''This is a repository for MongoDb.'''
def
__init__
(
self
):
super
()
.
__init__
(
netconst
.
BUSINESS_LOGIC_DB_HOSTNAME
,
netconst
.
BUSINESS_LOGIC_DB_PORT
,
'business-logic-db'
)
self
.
_adapter_collection
=
'layer_adapters'
self
.
_use_case_collection
=
'use_cases'
def
all
(
self
)
->
List
[
Dict
]:
result
=
super
()
.
get_entries
(
self
.
_adapter_collection
,
projection
=
{
'_id'
:
False
})
return
[
LayerAdapter
.
from_serializable_dict
(
row
)
for
row
in
list
(
result
)]
def
all_for_use_case
(
self
,
use_case
:
str
)
->
List
[
LayerAdapter
]:
result
=
super
()
.
get_entries
(
self
.
_adapter_collection
,
projection
=
{
'_id'
:
False
},
selection
=
{
"use_case"
:
use_case
})
return
[
LayerAdapter
.
from_serializable_dict
(
row
)
for
row
in
list
(
result
)]
def
all_for_use_case_and_table
(
self
,
use_case
:
str
,
table
:
str
)
->
List
[
LayerAdapter
]:
result
=
super
()
.
get_entries
(
self
.
_adapter_collection
,
projection
=
{
'_id'
:
False
},
selection
=
{
"use_case"
:
use_case
,
"table"
:
table
})
return
[
LayerAdapter
.
from_serializable_dict
(
row
)
for
row
in
list
(
result
)]
def
one
(
self
,
name
:
str
,
use_case
:
str
,
table
:
str
)
->
LayerAdapter
:
result
=
list
(
super
()
.
get_entries
(
self
.
_adapter_collection
,
selection
=
{
"name"
:
name
,
"use_case"
:
use_case
,
"table"
:
table
}))
if
len
(
result
)
==
1
:
return
LayerAdapter
.
from_serializable_dict
(
result
[
0
])
return
None
def
delete_all
(
self
):
collection
=
self
.
_database
[
self
.
_adapter_collection
]
collection
.
delete_many
({})
def
add
(
self
,
adapter
:
LayerAdapter
):
super
()
.
insert_entry
(
self
.
_adapter_collection
,
adapter
.
to_serializable_dict
())
def
update_use_case
(
self
,
adapter
:
LayerAdapter
,
use_case
:
str
):
collection
=
self
.
_database
[
self
.
_adapter_collection
]
collection
.
update_one
({
"name"
:
adapter
.
name
,
"use_case"
:
use_case
,
"table"
:
adapter
.
table
},
{
"$set"
:
adapter
.
to_serializable_dict
()})
def
update
(
self
,
adapter
:
LayerAdapter
):
collection
=
self
.
_database
[
self
.
_adapter_collection
]
collection
.
update_one
({
"name"
:
adapter
.
name
,
"use_case"
:
adapter
.
use_case
,
"table"
:
adapter
.
table
},
{
"$set"
:
adapter
.
to_serializable_dict
()})
def
delete
(
self
,
adapter
:
LayerAdapter
):
collection
=
self
.
_database
[
self
.
_adapter_collection
]
collection
.
delete_many
({
"name"
:
adapter
.
name
,
"use_case"
:
adapter
.
use_case
,
"table"
:
adapter
.
table
})
\ No newline at end of file
src/participation-hub/federated-learning-microservice/app/db/table_repository.py
deleted
100644 → 0
View file @
e1216230
# global imports (dont't worry, red is normal)
import
network_constants
as
netconst
from
database.MongoRepositoryBase
import
MongoRepositoryBase
from
db.entities.table
import
Table
from
typing
import
Dict
,
List
class
TableRepository
(
MongoRepositoryBase
):
def
__init__
(
self
):
super
()
.
__init__
(
netconst
.
BUSINESS_LOGIC_DB_HOSTNAME
,
netconst
.
BUSINESS_LOGIC_DB_PORT
,
'business-logic-db'
)
self
.
_collection
=
'tables'
def
get_all
(
self
)
->
List
[
Table
]:
result
=
super
()
.
get_entries
(
self
.
_collection
,
projection
=
{
'_id'
:
False
})
return
[
Table
.
from_serializable_dict
(
row
)
for
row
in
list
(
result
)]
def
get_all_for_use_case
(
self
,
use_case
:
str
)
->
List
[
Table
]:
result
=
super
()
.
get_entries
(
self
.
_collection
,
selection
=
{
'use_case'
:
use_case
},
projection
=
{
'_id'
:
False
})
return
[
Table
.
from_serializable_dict
(
row
)
for
row
in
result
]
def
get_for_use_case_and_name
(
self
,
use_case
:
str
,
name
:
str
)
->
Table
:
result
=
list
(
super
()
.
get_entries
(
self
.
_collection
,
selection
=
{
"use_case"
:
use_case
,
"name"
:
name
},
projection
=
{
'_id'
:
False
}))
if
len
(
result
)
==
1
:
return
Table
.
from_serializable_dict
(
result
[
0
])
if
len
(
result
)
==
0
:
return
None
raise
ValueError
(
"No more than 1 Schema allowed per use-case!"
)
def
add
(
self
,
table
:
Table
):
super
()
.
insert_entry
(
self
.
_collection
,
table
.
to_serializable_dict
())
def
delete_for_use_case
(
self
,
use_case
:
str
):
collection
=
self
.
_database
[
self
.
_collection
]
collection
.
delete_many
({
"use_case"
:
use_case
})
def
delete_all
(
self
):
collection
=
self
.
_database
[
self
.
_collection
]
collection
.
delete_many
({})
def
update
(
self
,
table
:
Table
):
collection
=
self
.
_database
[
self
.
_collection
]
collection
.
update_one
({
"use_case"
:
table
.
use_case
,
"name"
:
table
.
name
},
{
"$set"
:
table
.
to_serializable_dict
()})
\ No newline at end of file
src/participation-hub/federated-learning-microservice/app/db/use_case_repository.py
deleted
100644 → 0
View file @
e1216230
# global imports (dont't worry, red is normal)
import
network_constants
as
netconst
from
database.MongoRepositoryBase
import
MongoRepositoryBase
from
db.entities.use_case
import
UseCase
import
pymongo
import
json
from
typing
import
List
,
Dict
class
UseCaseRepository
(
MongoRepositoryBase
):
'''This is a repository for MongoDb.'''
def
__init__
(
self
):
super
()
.
__init__
(
netconst
.
BUSINESS_LOGIC_DB_HOSTNAME
,
netconst
.
BUSINESS_LOGIC_DB_PORT
,
'business-logic-db'
)
self
.
_use_case_collection
=
'use_cases'
def
all
(
self
)
->
List
[
UseCase
]:
dicts
=
list
(
super
()
.
get_entries
(
self
.
_use_case_collection
,
projection
=
{
'_id'
:
False
}))
return
[
UseCase
.
from_serializable_dict
(
d
)
for
d
in
dicts
]
def
delete_all
(
self
):
collection
=
self
.
_database
[
self
.
_use_case_collection
]
collection
.
delete_many
({})
def
delete_all_with_name
(
self
,
name
:
str
):
collection
=
self
.
_database
[
self
.
_use_case_collection
]
collection
.
delete_many
({
"name"
:
name
})
def
get_by_name
(
self
,
name
:
str
):
result
=
list
(
super
()
.
get_entries
(
self
.
_use_case_collection
,
{
"name"
:
name
}))
if
len
(
result
)
==
1
:
return
UseCase
.
from_serializable_dict
(
result
[
0
])
if
len
(
result
)
==
0
:
return
None
raise
ValueError
(
"More than one Use-Case in the DB!"
)
def
put
(
self
,
use_case_name
:
str
):
use_cases
=
self
.
all
()
existing_use_cases
=
list
(
filter
(
lambda
use_case
:
use_case
.
name
==
use_case_name
,
use_cases
))
if
len
(
existing_use_cases
)
==
0
:
use_case
=
UseCase
(
use_case_name
)
super
()
.
insert_entry
(
self
.
_use_case_collection
,
use_case
.
to_serializable_dict
())
\ No newline at end of file
src/participation-hub/federated-learning-microservice/app/main.py
View file @
fec9c13c
...
...
@@ -17,8 +17,6 @@ from flask import redirect
app
=
connexion
.
App
(
__name__
,
specification_dir
=
'configs/'
)
from
db.entities.layer_adapter
import
LayerAdapter
@
app
.
app
.
before_request
def
before_request
():
if
request
.
url
.
startswith
(
'http://'
):
...
...
src/participation-hub/federated-learning-microservice/app/processing/
test
/__init__.py
→
src/participation-hub/federated-learning-microservice/app/processing/
crowd_journalism
/__init__.py
View file @
fec9c13c
File moved
src/participation-hub/federated-learning-microservice/app/processing/
test
/checkpoint_manager.py
→
src/participation-hub/federated-learning-microservice/app/processing/
crowd_journalism
/checkpoint_manager.py
View file @
fec9c13c
File moved
src/participation-hub/federated-learning-microservice/app/processing/crowd_journalism/federated_algorithm.py
0 → 100644
View file @
fec9c13c
#import processing.text_processing.global_hyperparams as globals
import
global_hyperparams
as
globals
from
model
import
get_simple_LSTM_model
import
pandas
as
pd
from
sklearn.model_selection
import
train_test_split
from
tensorflow.keras.models
import
Sequential
from
tensorflow.keras.layers
import
LSTM
,
Dense
,
Dropout
,
Embedding
from
tensorflow.keras.preprocessing.text
import
Tokenizer
from
tensorflow.keras.preprocessing.sequence
import
pad_sequences
import
collections
import
numpy
as
np
import
tensorflow
as
tf
import
tensorflow_federated
as
tff
def
model_fn
():
keras_model
=
get_simple_LSTM_model
()
#return tff.learning.from_compiled_keras_model(keras_model, sample_batch) original
return
tff
.
learning
.
from_keras_model
(
keras_model
,
input_spec
=
globals
.
INPUT_SPEC
,
loss
=
tf
.
keras
.
losses
.
SparseCategoricalCrossentropy
(
from_logits
=
True
),
metrics
=
[
tf
.
keras
.
metrics
.
SparseCategoricalAccuracy
()])
def
federated_computation_new
(
train_dataset
,
test_dataset
):
if
(
globals
.
INPUT_SPEC
==
None
):
#should never reach this place because INPUT_SPEC is instantiated inside get_preprocessed_train_test_data.
#however, if in the future, processed data is provided without hte preprocessing function it will be none -> therefore assign it here
globals
.
INPUT_SPEC
=
train_dataset
[
0
]
.
element_spec
# Training and evaluating the model
iterative_process
=
tff
.
learning
.
build_federated_averaging_process
(
model_fn
,
client_optimizer_fn
=
lambda
:
tf
.
keras
.
optimizers
.
SGD
(
lr
=
0.5
))
state
=
iterative_process
.
initialize
()
print
(
type
(
state
))
for
n
in
range
(
globals
.
EPOCHS
):
state
,
metrics
=
iterative_process
.
next
(
state
,
train_dataset
)
print
(
'round {}, training metrics={}'
.
format
(
n
+
1
,
metrics
))
evaluation
=
tff
.
learning
.
build_federated_evaluation
(
model_fn
)
eval_metrics
=
evaluation
(
state
.
model
,
train_dataset
)
print
(
'Training evaluation metrics={}'
.
format
(
eval_metrics
))
test_metrics
=
evaluation
(
state
.
model
,
test_dataset
)
print
(
'Test evaluation metrics={}'
.
format
(
test_metrics
))
return
state
,
metrics
###############################################################################################
def
federated_computation_continue
(
train_dataset
,
test_dataset
,
restored_state
):
if
(
globals
.
INPUT_SPEC
==
None
):
#should never reach this place because INPUT_SPEC is instantiated inside get_preprocessed_train_test_data.
#however, if in the future, processed data is provided without hte preprocessing function it will be none -> therefore assign it here
globals
.
INPUT_SPEC
=
train_dataset
[
0
]
.
element_spec
# Training and evaluating the model
iterative_process
=
tff
.
learning
.
build_federated_averaging_process
(
model_fn
,
client_optimizer_fn
=
lambda
:
tf
.
keras
.
optimizers
.
SGD
(
lr
=
0.5
))
state
=
iterative_process
.
initialize
()
state
=
restored_state
print
(
type
(
state
))
for
n
in
range
(
globals
.
EPOCHS
):
state
,
metrics
=
iterative_process
.
next
(
state
,
train_dataset
)
print
(
'round {}, training metrics={}'
.
format
(
n
+
1
,
metrics
))
evaluation
=
tff
.
learning
.
build_federated_evaluation
(
model_fn
)
eval_metrics
=
evaluation
(
state
.
model
,
train_dataset
)
print
(
'Training evaluation metrics={}'
.
format
(
eval_metrics
))
test_metrics
=
evaluation
(
state
.
model
,
test_dataset
)
print
(
'Test evaluation metrics={}'
.
format
(
test_metrics
))
return
state
,
metrics
###############################################################################################
src/participation-hub/federated-learning-microservice/app/processing/
test
/main_proc.py
→
src/participation-hub/federated-learning-microservice/app/processing/
crowd_journalism
/main_proc.py
View file @
fec9c13c
File moved
src/participation-hub/federated-learning-microservice/app/processing/text_processing/ledger.csv
View file @
fec9c13c
Trainer_id,Model_id,Dataset_id,Accuracy,Loss
0,1623766462,1623766462,0.5,nan
0,1623768325,1623768325,0.25,nan
1,1623768510,1623768510,0.75,nan
src/participation-hub/federated-learning-microservice/app/processing/text_processing/models/ckpt_1623768325/saved_model.pb
0 → 100644
View file @
fec9c13c
File added
src/participation-hub/federated-learning-microservice/app/processing/text_processing/models/ckpt_1623768325/variables/variables.data-00000-of-00001
0 → 100644
View file @
fec9c13c
File added
src/participation-hub/federated-learning-microservice/app/processing/text_processing/models/ckpt_1623768325/variables/variables.index
0 → 100644
View file @
fec9c13c
File added
src/participation-hub/federated-learning-microservice/app/processing/text_processing/models/ckpt_1623768510/saved_model.pb
0 → 100644
View file @
fec9c13c
File added
src/participation-hub/federated-learning-microservice/app/processing/text_processing/models/ckpt_1623768510/variables/variables.data-00000-of-00001
0 → 100644
View file @
fec9c13c
File added
src/participation-hub/federated-learning-microservice/app/processing/text_processing/models/ckpt_1623768510/variables/variables.index
0 → 100644
View file @
fec9c13c
File added
src/participation-hub/federated-learning-microservice/app/routes/developers.py
View file @
fec9c13c
...
...
@@ -52,5 +52,5 @@ def trainMetricsToJSON(last_train_metrics : list):
metricsDict
[
"Loss"
]
=
last_train_metrics
[
4
]
return
json
.
dumps
(
metricsDict
)
#
upload_and_train("text_processing",1)
upload_and_train
(
"text_processing"
,
1
)
src/participation-hub/federated-learning-microservice/app/routes/layer.py
deleted
100644 → 0
View file @
e1216230
#global imports
from
db.entities.layer_adapter
import
LayerAdapter
from
db.repository
import
Repository
from
db.table_repository
import
TableRepository
from
db.use_case_repository
import
UseCaseRepository
from
services.layer_adapter_service
import
LayerAdapterService
import
json
from
flask
import
Response
,
request
table_repository
=
TableRepository
()
layer_repository
=
Repository
()
use_case_repository
=
UseCaseRepository
()
def
all
():
return
[
layer
.
to_serializable_dict
()
for
layer
in
layer_repository
.
all
()]
def
delete_all_layers
():
'''
delete all layers from the DB
'''
layer_repository
.
delete_all
()
return
Response
(
status
=
200
)
def
add_complete
():
'''
add a layer already containing attribute mappings to the BD.
'''
data
=
request
.
json
if
"name"
not
in
data
or
"properties"
not
in
data
or
"cluster_properties"
not
in
data
or
"use_case"
not
in
data
or
"table"
not
in
data
:
return
Response
(
status
=
400
,
response
=
f
"Field missing! Fields required: (name, properties, cluster_properties, table), present:({data.keys()})"
)
use_case
=
data
[
"use_case"
]
table_name
=
data
[
"table"
]
layer_name
=
data
[
"name"
]
table
=
table_repository
.
get_for_use_case_and_name
(
use_case
,
table_name
)
if
table
==
None
:
return
Response
(
status
=
400
,
response
=
"Table does not exist!"
)
layer
=
layer_repository
.
one
(
layer_name
,
use_case
,
table
.
name
)
if
layer
!=
None
:
return
Response
(
status
=
400
,
response
=
f
'Layer with name "{data["name"]}" already exists!'
)
# check if table contains mapping
for
prop
in
data
[
"properties"
]:
if
prop
not
in
table
.
mappings
.
keys
():
return
Response
(
status
=
400
,
response
=
f
"'{prop}' is no property of the Table!"
)
try
:
layer_new
=
LayerAdapter
.
from_serializable_dict
(
data
)
except
BaseException
as
e
:
print
(
f
"Exception: {e}"
)
return
Response
(
status
=
400
,
response
=
f
"{e}"
)
try
:
LayerAdapterService
.
add_complete
(
layer_new
)
except
ValueError
as
e
:
return
Response
(
status
=
400
,
response
=
f
"{e}"
)
return
Response
(
status
=
200
)
def
add_cluster_mapping
(
use_case
:
str
,
table
:
str
,
name
:
str
):
'''
add a mapped property to the list of properties to cluster with
@params:
use_case - Required : String-identifier for the Use-Case the Layer belongs to
table - Required : unique identifier of the Table the Layer belongs to
name - Required : unique identifier for the Layer
'''
layer
=
layer_repository
.
one
(
name
,
use_case
,
table
)
if
layer
==
None
:
return
Response
(
status
=
404
,
response
=
f
"Layer with name '{name}' does not exist!"
)
data
=
request
.
json
if
"attribute"
not
in
data
:
return
Response
(
status
=
400
,
response
=
f
"Field missing! Fields required: (attribute)"
)
try
:
layer
.
add_cluster_mapping
(
data
[
"attribute"
])
layer_repository
.
update
(
layer
)
return
Response
(
status
=
200
)
except
:
return
Response
(
status
=
400
,
response
=
f
'{data["attribute"]} is no attribute in the layer!'
)
def
delete_cluster_mapping
(
use_case
:
str
,
table
:
str
,
name
:
str
):
'''
remove a mapped property from the list of properties to cluster with
@params:
use_case - Required : String-identifier for the Use-Case the Layer belongs to
table - Required : unique identifier of the Table the Layer belongs to
name - Required : unique identifier for the Layer
'''
layer
=
layer_repository
.
one
(
name
,
use_case
,
table
)
if
layer
==
None
:
return
Response
(
status
=
404
,
response
=
f
"Layer with name '{name}' does not exist!"
)
data
=
request
.
json
if
"attribute"
not
in
data
:
return
Response
(
status
=
400
,
response
=
f
"Field missing! Fields required: (attribute)"
)
try
:
layer
.
delete_cluster_mapping
(
data
[
"attribute"
])
layer_repository
.
update
(
layer
)
return
Response
(
status
=
200
)
except
ValueError
as
e
:
print
(
e
)
return
Response
(
status
=
400
,
response
=
f
'{data["attribute"]} is no attribute in the layer!'
)
def
get_all_for_use_case
(
use_case
:
str
):
'''
get all layers assigned to the given use_case
'''
use_case_repository
.
put
(
use_case
)
return
[
layer
.
to_serializable_dict
()
for
layer
in
layer_repository
.
all_for_use_case
(
use_case
)]
def
one
(
use_case
:
str
,
table
:
str
,
name
:
str
):
'''
fetch a single layer from the DB
@params:
use_case - Required : String-identifier for the Use-Case the Layer belongs to
table - Required : unique identifier of the Table the Layer belongs to
name - Required : unique identifier for the Layer
'''
layer
=
layer_repository
.
one
(
name
,
use_case
,
table
)
if
layer
==
None
:
return
Response
(
status
=
404
,
response
=
f
"Layer with name '{name}' does not exist!"
)
return
Response
(
status
=
200
,
response
=
json
.
dumps
(
layer
.
to_serializable_dict
()))
def
delete_mapping
(
use_case
:
str
,
table
:
str
,
name
:
str
):
'''
delete a mapping from the layer identified by the internal representation
@params:
use_case - Required : String-identifier for the Use-Case the Layer belongs to
table - Required : unique identifier of the Table the Layer belongs to
name - Required : unique identifier for the Layer
'''
use_case_repository
.
put
(
use_case
)
layer
=
layer_repository
.
one
(
name
,
use_case
,
table
)
if
layer
==
None
:
return
Response
(
status
=
404
,
response
=
f
"Layer with name '{name}' does not exist!"
)
data
=
request
.
json
if
"internal"
not
in
data
:
return
Response
(
status
=
400
,
response
=
f
"Field missing! Fields required: (internal)"
)
try
:
layer
.
delete_mapping
(
data
[
"internal"
])
layer_repository
.
update
(
layer
)
except
ValueError
:
return
Response
(
status
=
400
,
response
=
f
'{data["internal"]} is not a property of the layer!'
)
return
Response
(
status
=
200
)
def
add_mapping
(
name
:
str
,
table
:
str
,
use_case
:
str
):
'''
add a new mapping to the layer identified by name
@params:
use_case - Required : String-identifier for the Use-Case the Layer belongs to
table - Required : unique identifier of the Table the Layer belongs to
name - Required : unique identifier for the Layer
'''
use_case_repository
.
put
(
use_case
)
layer
=
layer_repository
.
one
(
name
,
use_case
)
if
layer
==
None
:
return
Response
(
status
=
404
,
response
=
f
"Layer with name '{name}' does not exist!"
)
data
=
request
.
json
if
"internal"
not
in
data
:
return
Response
(
status
=
400
,
response
=
f
"Field missing! Fields required: (internal)"
)
# check if schema contains mapping
table
=
table_repository
.
get_for_use_case_and_name
(
use_case
,
table
)
if
table
==
None
:
return
Response
(
status
=
400
,
response
=
f
'Table does not exist.'
)
if
data
[
"internal"
]
not
in
table
.
mappings
:
return
Response
(
status
=
400
,
response
=
f
'{data["internal"]} is not existent in the table!'
)
layer
.
add_mapping
(
data
[
"internal"
])
layer_repository
.
update
(
layer
)
return
Response
(
status
=
200
)
def
delete_one
(
use_case
:
str
,
table
:
str
,
name
:
str
):
'''
delete a layer and all its mappings from the Db
@params:
use_case - Required : String-identifier for the Use-Case the Layer belongs to
table - Required : unique identifier of the Table the Layer belongs to
name - Required : unique identifier for the Layer
'''
layer
=
layer_repository
.
one
(
name
,
use_case
,
table
)
if
layer
==
None
:
return
Response
(
status
=
404
,
response
=
f
"Layer with name '{name}' does not exist!"
)
layer_repository
.
delete
(
layer
)
return
Response
(
status
=
200
)
def
get_layers_for_use_case_and_table
(
use_case
:
str
,
table
:
str
):
return
[
layer
.
to_serializable_dict
()
for
layer
in
layer_repository
.
all_for_use_case_and_table
(
use_case
,
table
)]
\ No newline at end of file
src/participation-hub/federated-learning-microservice/app/routes/owners.py
View file @
fec9c13c
...
...
@@ -74,6 +74,7 @@ def trainMetricsToJSON(last_train_metrics : list):
metricsDict
[
"Accuracy"
]
=
last_train_metrics
[
3
]
metricsDict
[
"Loss"
]
=
last_train_metrics
[
4
]
return
json
.
dumps
(
metricsDict
)
#last("text_processing")
#upload_and_train("text_processing") #warning it deletes the files
\ No newline at end of file
upload_and_train
(
"text_processing"
)
#warning it deletes the files
last
(
"text_processing"
)
\ No newline at end of file
src/participation-hub/federated-learning-microservice/app/routes/tables.py
deleted
100644 → 0
View file @
e1216230
#global imports
from
db.table_repository
import
TableRepository
from
db.use_case_repository
import
UseCaseRepository
from
db.entities.table
import
Table
from
flask
import
Response
,
request
table_repository
=
TableRepository
()
use_case_repository
=
UseCaseRepository
()
def
all
():
return
[
t
.
to_serializable_dict
()
for
t
in
table_repository
.
get_all
()]
def
all_for_use_case
(
use_case
:
str
):
return
[
t
.
to_serializable_dict
()
for
t
in
table_repository
.
get_all_for_use_case
(
use_case
)]
def
put_mapping
(
use_case
:
str
,
name
:
str
):
body
=
request
.
json
if
"internal"
not
in
body
or
"external"
not
in
body
:
return
Response
(
status
=
400
,
response
=
f
"Field missing! Fields required: (internal, external)"
)
table
=
table_repository
.
get_for_use_case_and_name
(
use_case
,
name
)
if
table
==
None
:
print
(
"table not there, creating it..."
)
table
=
table_repository
.
add
(
Table
(
use_case
,
name
,
{}))
table
.
add_mapping
(
body
[
"internal"
],
body
[
"external"
])
table_repository
.
update
(
table
)
return
Response
(
status
=
200
)
def
add_complete
(
use_case
:
str
):
body
=
request
.
json
body
[
"use_case"
]
=
use_case
# check if fields are present
if
"name"
not
in
body
or
"mappings"
not
in
body
:
return
Response
(
status
=
400
,
response
=
"Field missing! Fields required: (name, mappings)"
)
# check if table exists
table_reference
=
table_repository
.
get_for_use_case_and_name
(
body
[
"use_case"
],
body
[
"name"
])
if
table_reference
!=
None
:
return
Response
(
status
=
400
,
response
=
"Table already exists!"
)
use_case_repository
.
put
(
body
[
"use_case"
])
table_new
=
Table
.
from_serializable_dict
(
body
)
table_repository
.
add
(
table_new
)
return
Response
(
status
=
200
)
def
delete_all_for_use_case
(
use_case
:
str
):
table_repository
.
delete_for_use_case
(
use_case
)
return
Response
(
status
=
200
)
def
delete_all
():
table_repository
.
delete_all
()
return
Response
(
status
=
200
)
\ No newline at end of file
src/participation-hub/federated-learning-microservice/app/routes/use_case.py
deleted
100644 → 0
View file @
e1216230
# global imports
from
db.entities.layer_adapter
import
LayerAdapter
from
db.use_case_repository
import
UseCaseRepository
from
db.table_repository
import
TableRepository
from
db.repository
import
Repository
from
services.layer_adapter_service
import
LayerAdapterService
import
json
from
flask
import
Response
,
request
use_case_repository
=
UseCaseRepository
()
table_repository
=
TableRepository
()
repository
=
Repository
()
def
all
():
return
[
use_case
.
to_serializable_dict
()
for
use_case
in
use_case_repository
.
all
()]
def
delete_all
():
use_case_repository
.
delete_all
()
repository
.
delete_all
()
table_repository
.
delete_all
()
return
Response
(
status
=
200
)
def
add
():
body
=
request
.
json
if
"name"
not
in
body
.
keys
():
return
Response
(
status
=
400
,
response
=
"Field missing! Fields required: (name)"
)
name
=
body
[
"name"
]
# check if use-case exists
reference
=
use_case_repository
.
get_by_name
(
name
)
if
not
reference
==
None
:
return
Response
(
status
=
400
,
response
=
"Use-Case already exists!"
)
use_case_repository
.
put
(
name
)
return
Response
(
status
=
200
)
src/participation-hub/federated-learning-microservice/app/tests/test_layer_adapter.py
deleted
100644 → 0
View file @
e1216230
import
unittest
import
manage_sys_paths
from
db.entities.layer_adapter
import
LayerAdapter
class
Test_Layer_Adapter
(
unittest
.
TestCase
):
def
test_LayerAdapter_newLayerAdapterObj_validInstantiation
(
self
):
adapter1
=
LayerAdapter
(
"layer1"
,
"use_case"
,
"table"
,
[
"a"
,
"c"
],
[
"a"
])
print
(
adapter1
.
to_serializable_dict
)
if
__name__
==
'__main__'
:
unittest
.
main
()
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment