Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
S
SMART
Project
Project
Details
Activity
Releases
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
3
Issues
3
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Registry
Registry
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
UNI-KLU
SMART
Commits
9f4db2a6
Commit
9f4db2a6
authored
Oct 15, 2020
by
Bogdan
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Modified a few files for visualisation
parent
e359a787
Changes
7
Show whitespace changes
Inline
Side-by-side
Showing
7 changed files
with
204 additions
and
1 deletion
+204
-1
run_clustering.py
...b/role-stage-discovery-microservice/app/run_clustering.py
+18
-0
run_similarity_calc.py
...e-stage-discovery-microservice/app/run_similarity_calc.py
+15
-0
visualisationPaper.py
...le-stage-discovery-microservice/app/visualisationPaper.py
+54
-0
routes.yml
...-hub/semantic-linking-microservice/app/configs/routes.yml
+80
-0
repository.py
...ta-hub/semantic-linking-microservice/app/db/repository.py
+16
-0
layers.py
...ta-hub/semantic-linking-microservice/app/routes/layers.py
+10
-0
nodes.py
...ata-hub/semantic-linking-microservice/app/routes/nodes.py
+11
-1
No files found.
src/data-hub/role-stage-discovery-microservice/app/run_clustering.py
View file @
9f4db2a6
...
@@ -17,6 +17,24 @@ repo = Repository()
...
@@ -17,6 +17,24 @@ repo = Repository()
def
run_generic_clustering
():
def
run_generic_clustering
():
'''Runs the clustering for all layers found in the repository.'''
'''Runs the clustering for all layers found in the repository.'''
all_layers
:
List
[
Layer
]
=
repo
.
get_layers
()
all_layers
:
List
[
Layer
]
=
repo
.
get_layers
()
#all_layers:List[Layer] = repo.get_layers_for_use_case("paper")
for
layer
in
all_layers
:
print
(
f
"Clustering {layer.use_case}//{layer.use_case_table}//{layer.layer_name}."
)
if
layer
.
properties
is
None
or
len
(
layer
.
properties
)
==
0
:
print
(
"skipping"
)
continue
try
:
clusters
=
run_clustering_for_layer
(
layer
)
store_generic_clusters
(
clusters
)
except
Exception
as
e
:
print
(
str
(
e
))
def
run_clustering_for_Paper_case
():
all_layers
:
List
[
Layer
]
=
repo
.
get_layers_for_use_case
(
"paper"
)
for
layer
in
all_layers
:
for
layer
in
all_layers
:
print
(
f
"Clustering {layer.use_case}//{layer.use_case_table}//{layer.layer_name}."
)
print
(
f
"Clustering {layer.use_case}//{layer.use_case_table}//{layer.layer_name}."
)
...
...
src/data-hub/role-stage-discovery-microservice/app/run_similarity_calc.py
View file @
9f4db2a6
...
@@ -6,6 +6,7 @@ repo = Repository()
...
@@ -6,6 +6,7 @@ repo = Repository()
def
run_similarity_calc_per_use_case
():
def
run_similarity_calc_per_use_case
():
layers
=
repo
.
get_layers
()
layers
=
repo
.
get_layers
()
#layers = repo.get_layers_for_use_case("paper")
uc_layers
=
{}
uc_layers
=
{}
for
layer
in
layers
:
for
layer
in
layers
:
...
@@ -19,6 +20,20 @@ def run_similarity_calc_per_use_case():
...
@@ -19,6 +20,20 @@ def run_similarity_calc_per_use_case():
print
(
f
"Running for use case {key} with layers {str(layers)}."
)
print
(
f
"Running for use case {key} with layers {str(layers)}."
)
SimilarityCalc
.
main
(
layerNameList
=
layers
)
SimilarityCalc
.
main
(
layerNameList
=
layers
)
def
run_similarity_calc_for_Paper_case
():
layers
=
repo
.
get_layers_for_use_case
(
"paper"
)
uc_layers
=
{}
for
layer
in
layers
:
uc
=
layer
.
use_case
if
uc
not
in
uc_layers
:
uc_layers
[
uc
]
=
[]
uc_layers
[
uc
]
.
append
(
layer
.
layer_name
)
for
key
in
uc_layers
:
layers
=
uc_layers
[
key
]
print
(
f
"Running for use case {key} with layers {str(layers)}."
)
SimilarityCalc
.
main
(
layerNameList
=
layers
)
if
__name__
==
'__main__'
:
if
__name__
==
'__main__'
:
run_similarity_calc_per_use_case
()
run_similarity_calc_per_use_case
()
\ No newline at end of file
src/data-hub/role-stage-discovery-microservice/app/visualisationPaper.py
0 → 100644
View file @
9f4db2a6
# add modules folder to interpreter path
# import sys
# import os
# modules_path = '../../../modules/'
# if os.path.exists(modules_path):
# sys.path.insert(1, modules_path)
# #######LOGGING############
# import logging
# LOG_FORMAT = ('%(levelname) -5s %(asctime)s %(name)s:%(funcName) -35s %(lineno) -5d: %(message)s')
# logging.basicConfig(level=logging.INFO, format=LOG_FORMAT)
# LOGGER = logging.getLogger(__name__)
######SECURITY#############
# import connecion
# from security import swagger_util
# from pathlib import Path
# from env_info import is_running_locally, get_resources_path
######PROCESSING##########
#from run_clustering import run_clustering_for_Paper_case
import
run_clustering
as
clustering
import
run_similarity_calc
as
similCalc
import
processing.similarityFiles.dataInput
as
inputData
##################################
#TODO: get certificates/authorization
# SSL configuration
# certificate_path = get_resources_path()
# context = (os.path.normpath(f'{certificate_path}/articonf1.crt'), os.path.normpath(f'{certificate_path}/articonf1.key')) # certificate and key files
# if is_running_locally():
# print("Running locally...")
# app.add_api(swagger_util.get_bundled_specs(Path("configs/swagger_local.yml")),
# resolver = connexion.RestyResolver("cms_rest_api"))
# else:
# app.add_api(swagger_util.get_bundled_specs(Path("configs/swagger.yml")),
# resolver = connexion.RestyResolver("cms_rest_api"))
##################################
def
mainViz
():
#procesing data
clustering
.
run_clustering_for_Paper_case
()
similCalc
.
run_similarity_calc_for_Paper_case
()
inputData
.
getClusterDataFromMongo
(
"Paper"
,
None
,
None
)
#inputData.getSimilarityDataFromMongo(cluster_layer: str= None, batchSize: int=1000, run_id: str=None)
#similarityArrFromMongo = getSimilarityDataFromMongo("Paper")
src/data-hub/semantic-linking-microservice/app/configs/routes.yml
View file @
9f4db2a6
...
@@ -38,6 +38,23 @@ paths:
...
@@ -38,6 +38,23 @@ paths:
description
:
"
Successful
operation"
description
:
"
Successful
operation"
schema
:
schema
:
$ref
:
"
#/definitions/NodeCollection"
$ref
:
"
#/definitions/NodeCollection"
delete
:
security
:
-
JwtAdmin
:
[]
operationId
:
"
routes.nodes.delete_nodes_for_use_case"
tags
:
-
"
Nodes"
summary
:
"
Delete
all
nodes
in
the
Use-Case"
description
:
"
Delete
all
Nodes
in
the
specified
Use-case"
parameters
:
-
name
:
"
use_case"
in
:
"
path"
description
:
"
Name
of
the
requested
Use-Case"
required
:
true
type
:
"
string"
responses
:
'
200'
:
description
:
"
Successful
Request"
/use-cases/{use_case}/tables/{table_name}/nodes
:
/use-cases/{use_case}/tables/{table_name}/nodes
:
get
:
get
:
security
:
security
:
...
@@ -62,6 +79,30 @@ paths:
...
@@ -62,6 +79,30 @@ paths:
description
:
"
Successful
operation"
description
:
"
Successful
operation"
schema
:
schema
:
$ref
:
"
#/definitions/NodeCollection"
$ref
:
"
#/definitions/NodeCollection"
delete
:
security
:
-
JwtAdmin
:
[]
operationId
:
"
routes.nodes.delete_nodes_for_table"
tags
:
-
"
Nodes"
summary
:
"
Delete
all
nodes
in
the
table
of
the
Use-Case"
description
:
"
Delete
all
Nodes
in
the
table
of
the
specified
Use-case"
parameters
:
-
name
:
"
use_case"
in
:
"
path"
description
:
"
Name
of
the
requested
Use-Case"
required
:
true
type
:
"
string"
-
name
:
"
table_name"
in
:
"
path"
description
:
"
Name
of
Table
the
node
belongs
to"
required
:
true
type
:
"
string"
responses
:
'
200'
:
description
:
"
Successful
Request"
/use-cases/{use_case}/tables/{table_name}/layers/{layer_name}/nodes
:
/use-cases/{use_case}/tables/{table_name}/layers/{layer_name}/nodes
:
get
:
get
:
security
:
security
:
...
@@ -132,6 +173,28 @@ paths:
...
@@ -132,6 +173,28 @@ paths:
description
:
"
Successful
operation"
description
:
"
Successful
operation"
schema
:
schema
:
$ref
:
"
#/definitions/LayerCollection"
$ref
:
"
#/definitions/LayerCollection"
delete
:
security
:
-
JwtAdmin
:
[]
operationId
:
"
routes.layers.delete_layers_for_table"
tags
:
-
"
Layers"
summary
:
"
Delete
all
layers
in
the
table
of
the
Use-Case"
description
:
"
Delete
all
layers
in
the
table
of
the
specified
Use-case"
parameters
:
-
name
:
"
use_case"
in
:
"
path"
description
:
"
Name
of
the
requested
Use-Case"
required
:
true
type
:
"
string"
-
name
:
"
table_name"
in
:
"
path"
description
:
"
Name
of
Table"
required
:
true
type
:
"
string"
responses
:
'
200'
:
description
:
"
Successful
Request"
/use-cases/{use_case}/tables/{table_name}/layers/{name}
:
/use-cases/{use_case}/tables/{table_name}/layers/{name}
:
...
@@ -187,6 +250,23 @@ paths:
...
@@ -187,6 +250,23 @@ paths:
$ref
:
"
#/definitions/LayerCollection"
$ref
:
"
#/definitions/LayerCollection"
'
404'
:
'
404'
:
description
:
"
No
layers
found"
description
:
"
No
layers
found"
delete
:
security
:
-
JwtAdmin
:
[]
operationId
:
"
routes.layers.delete_layers_for_use_case"
tags
:
-
"
Layers"
summary
:
"
Delete
all
layers
in
the
Use-Case"
description
:
"
Delete
all
layers
in
the
specified
Use-case"
parameters
:
-
name
:
"
use_case"
in
:
"
path"
description
:
"
Name
of
the
requested
Use-Case"
required
:
true
type
:
"
string"
responses
:
'
200'
:
description
:
"
Successful
Request"
# endregion layers
# endregion layers
definitions
:
definitions
:
...
...
src/data-hub/semantic-linking-microservice/app/db/repository.py
View file @
9f4db2a6
...
@@ -66,6 +66,11 @@ class Repository(MongoRepositoryBase):
...
@@ -66,6 +66,11 @@ class Repository(MongoRepositoryBase):
collection
=
self
.
_database
[
self
.
_layer_collection
]
collection
=
self
.
_database
[
self
.
_layer_collection
]
collection
.
delete_many
({
"use_case"
:
use_case
})
collection
.
delete_many
({
"use_case"
:
use_case
})
def
delete_layers_for_table
(
self
,
use_case
:
str
,
table
:
str
):
''' Delete layers from the table of the given Use_case '''
collection
=
self
.
_database
[
self
.
_layer_collection
]
collection
.
delete_many
({
"use_case"
:
use_case
,
"table"
:
table
})
def
add_layer_node
(
self
,
node
:
dict
):
def
add_layer_node
(
self
,
node
:
dict
):
super
()
.
insert_entry
(
self
.
_layer_nodes_collection
,
node
)
super
()
.
insert_entry
(
self
.
_layer_nodes_collection
,
node
)
...
@@ -95,4 +100,15 @@ class Repository(MongoRepositoryBase):
...
@@ -95,4 +100,15 @@ class Repository(MongoRepositoryBase):
projection
=
{
'_id'
:
0
})
projection
=
{
'_id'
:
0
})
return
list
(
entries
)
return
list
(
entries
)
def
delete_nodes_for_use_case
(
self
,
use_case
):
'''Deletes All node of the given use_case'''
collection
=
self
.
_database
[
self
.
_layer_nodes_collection
]
collection
.
delete_many
({
"use_case"
:
use_case
})
def
delete_nodes_for_table
(
self
,
use_case
,
table_name
):
'''Deletes nodes from the given table of the use-case'''
collection
=
self
.
_database
[
self
.
_layer_nodes_collection
]
collection
.
delete_many
({
"use_case"
:
use_case
,
"table"
:
table_name
})
# endregion
# endregion
src/data-hub/semantic-linking-microservice/app/routes/layers.py
View file @
9f4db2a6
...
@@ -33,4 +33,14 @@ def get_by_use_case(use_case: str):
...
@@ -33,4 +33,14 @@ def get_by_use_case(use_case: str):
else
:
else
:
return
Response
(
status
=
404
)
return
Response
(
status
=
404
)
def
delete_layers_for_use_case
(
use_case
:
str
):
repo
.
delete_layers_for_use_case
(
use_case
)
print
(
"Deleted all layers from"
+
use_case
)
return
Response
(
status
=
200
)
def
delete_layers_for_table
(
use_case
:
str
,
table_name
:
str
):
repo
.
delete_layers_for_table
(
use_case
,
table_name
)
print
(
"Deleted all layers from: "
+
use_case
+
" , "
+
table_name
)
return
Response
(
status
=
200
)
#endregion
#endregion
src/data-hub/semantic-linking-microservice/app/routes/nodes.py
View file @
9f4db2a6
...
@@ -21,3 +21,13 @@ def nodes_for_use_case_and_table(use_case: str, table_name: str) -> List[Dict]:
...
@@ -21,3 +21,13 @@ def nodes_for_use_case_and_table(use_case: str, table_name: str) -> List[Dict]:
def
nodes_for_use_case_and_table_and_layer
(
layer_name
:
str
,
use_case
:
str
,
table_name
:
str
)
->
List
[
Dict
]:
def
nodes_for_use_case_and_table_and_layer
(
layer_name
:
str
,
use_case
:
str
,
table_name
:
str
)
->
List
[
Dict
]:
res
=
repo
.
get_nodes_for_use_case_and_table_and_layer
(
layer_name
,
use_case
,
table_name
)
res
=
repo
.
get_nodes_for_use_case_and_table_and_layer
(
layer_name
,
use_case
,
table_name
)
return
res
return
res
def
delete_nodes_for_use_case
(
use_case
:
str
):
repo
.
delete_nodes_from_use_case
(
use_case
)
print
(
"Deleted all nodes from"
+
use_case
)
return
Response
(
status
=
200
)
def
delete_nodes_for_table
(
use_case
:
str
,
table_name
:
str
):
repo
.
delete_nodes_for_table
(
use_case
,
table_name
)
print
(
"Deleted all nodes from: "
+
use_case
+
" , "
+
table_name
)
return
Response
(
status
=
200
)
\ No newline at end of file
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment