Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
S
SMART
Project
Project
Details
Activity
Releases
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
3
Issues
3
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Registry
Registry
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
UNI-KLU
SMART
Commits
9c4399c4
Commit
9c4399c4
authored
Oct 15, 2020
by
Bogdan
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Fix clustering/fetching.. added MongoRep funct
parent
051f6946
Changes
4
Hide whitespace changes
Inline
Side-by-side
Showing
4 changed files
with
35 additions
and
17 deletions
+35
-17
repository.py
...ub/role-stage-discovery-microservice/app/db/repository.py
+13
-1
fetching.py
...iscovery-microservice/app/processing/fetching/fetching.py
+6
-3
visualisationPaper.py
...le-stage-discovery-microservice/app/visualisationPaper.py
+13
-13
MongoRepositoryBase.py
src/modules/database/MongoRepositoryBase.py
+3
-0
No files found.
src/data-hub/role-stage-discovery-microservice/app/db/repository.py
View file @
9c4399c4
...
@@ -57,8 +57,14 @@ class Repository(MongoRepositoryBase):
...
@@ -57,8 +57,14 @@ class Repository(MongoRepositoryBase):
else
:
else
:
return
None
return
None
def
delete_all_layers
(
self
):
def
delete_all_layers
(
self
,
use_case
:
str
):
super
()
.
drop_collection
(
self
.
_layer_collection
)
super
()
.
drop_collection
(
self
.
_layer_collection
)
def
delete_use_case_layers
(
self
,
use_case
:
str
):
collection_name
=
self
.
_layer_collection
criteria
=
{
"use_case"
:
use_case
}
res
=
super
()
.
delete_entry
(
collection_name
,
criteria
,
True
)
print
(
"###REPOSITORY: res= "
+
str
(
res
))
#endregion Layers
#endregion Layers
...
@@ -78,6 +84,12 @@ class Repository(MongoRepositoryBase):
...
@@ -78,6 +84,12 @@ class Repository(MongoRepositoryBase):
entries
=
super
()
.
get_entries
(
self
.
_layer_nodes_collection
,
selection
=
{
'use_case'
:
use_case
,
'use_case_table'
:
use_case_table
,
'layer_name'
:
layer_name
},
projection
=
{
'_id'
:
0
})
entries
=
super
()
.
get_entries
(
self
.
_layer_nodes_collection
,
selection
=
{
'use_case'
:
use_case
,
'use_case_table'
:
use_case_table
,
'layer_name'
:
layer_name
},
projection
=
{
'_id'
:
0
})
return
[
e
for
e
in
entries
]
return
[
e
for
e
in
entries
]
def
delete_use_case_nodes
(
self
,
use_case
:
str
):
collection_name
=
self
.
_layer_nodes_collection
criteria
=
{
"use_case"
:
use_case
}
res
=
super
()
.
delete_entry
(
collection_name
,
criteria
,
True
)
print
(
"###REPOSITORY: res= "
+
str
(
res
))
#endregion
#endregion
#region Clusters
#region Clusters
...
...
src/data-hub/role-stage-discovery-microservice/app/processing/fetching/fetching.py
View file @
9c4399c4
...
@@ -89,11 +89,14 @@ def _fetch_nodes(use_case: str, table: str, layer_name: str) -> List[Dict]:
...
@@ -89,11 +89,14 @@ def _fetch_nodes(use_case: str, table: str, layer_name: str) -> List[Dict]:
def
fetch_nodes_from_semantic_linking
():
def
fetch_nodes_from_semantic_linking
():
'''Empties the db and inserts layers and nodes from BusinessLogic and SemanticLinking'''
'''Empties the db and inserts layers and nodes from BusinessLogic and SemanticLinking'''
repository
=
Repository
()
repository
=
Repository
()
repository
.
delete_all_layers
()
#
repository.delete_all_layers()
repository
.
delete_all_nodes
()
#
repository.delete_all_nodes()
repository
.
delete_use_case_layers
(
"paper"
)
repository
.
delete_use_case_nodes
(
"paper"
)
use_cases
=
_fetch_use_cases
()
use_cases
=
_fetch_use_cases
()
for
use_case
in
use_cases
:
for
use_case
in
[
uc
for
uc
in
use_cases
if
uc
==
'paper'
]
:
print
(
f
"Fetching for use-case {use_case}"
)
print
(
f
"Fetching for use-case {use_case}"
)
tables
=
_fetch_tables
(
use_case
)
tables
=
_fetch_tables
(
use_case
)
...
...
src/data-hub/role-stage-discovery-microservice/app/visualisationPaper.py
View file @
9c4399c4
# add modules folder to interpreter path
###add modules folder to interpreter path
# import sys
import
sys
# import os
import
os
# modules_path = '../../../modules/'
modules_path
=
'../../../modules/'
# if os.path.exists(modules_path):
if
os
.
path
.
exists
(
modules_path
):
# sys.path.insert(1, modules_path)
sys
.
path
.
insert
(
1
,
modules_path
)
# #######LOGGING############
# import logging
# LOG_FORMAT = ('%(levelname) -5s %(asctime)s %(name)s:%(funcName) -35s %(lineno) -5d: %(message)s')
# logging.basicConfig(level=logging.INFO, format=LOG_FORMAT)
# LOGGER = logging.getLogger(__name__)
######SECURITY#############
######SECURITY#############
# import connecion
# import connecion
...
@@ -18,9 +14,9 @@
...
@@ -18,9 +14,9 @@
######PROCESSING##########
######PROCESSING##########
#from run_clustering import run_clustering_for_Paper_case
import
run_clustering
as
clustering
import
run_clustering
as
clustering
import
run_similarity_calc
as
similCalc
import
run_similarity_calc
as
similCalc
import
processing.fetching.fetching
as
fetchy
import
processing.similarityFiles.dataInput
as
inputData
import
processing.similarityFiles.dataInput
as
inputData
...
@@ -47,9 +43,13 @@ import processing.similarityFiles.dataInput as inputData
...
@@ -47,9 +43,13 @@ import processing.similarityFiles.dataInput as inputData
def
mainViz
():
def
mainViz
():
#procesing data
#procesing data
clustering
.
run_clustering_for_Paper_case
()
#fetchy.fetch_nodes_from_semantic_linking()
#clustering.run_clustering_for_Paper_case()
similCalc
.
run_similarity_calc_for_Paper_case
()
similCalc
.
run_similarity_calc_for_Paper_case
()
inputData
.
getClusterDataFromMongo
(
"Paper"
,
None
,
None
)
inputData
.
getClusterDataFromMongo
(
"Paper"
,
None
,
None
)
#inputData.getSimilarityDataFromMongo(cluster_layer: str= None, batchSize: int=1000, run_id: str=None)
#inputData.getSimilarityDataFromMongo(cluster_layer: str= None, batchSize: int=1000, run_id: str=None)
#similarityArrFromMongo = getSimilarityDataFromMongo("Paper")
#similarityArrFromMongo = getSimilarityDataFromMongo("Paper")
mainViz
()
\ No newline at end of file
src/modules/database/MongoRepositoryBase.py
View file @
9c4399c4
...
@@ -15,6 +15,9 @@ class MongoRepositoryBase:
...
@@ -15,6 +15,9 @@ class MongoRepositoryBase:
def
drop_collection
(
self
,
collection_name
):
def
drop_collection
(
self
,
collection_name
):
self
.
_database
[
collection_name
]
.
drop
()
self
.
_database
[
collection_name
]
.
drop
()
def
delete_entry
(
self
,
collection_name
,
criteria
:
dict
,
multi
:
bool
):
self
.
_database
[
collection_name
]
.
remove
(
criteria
,
multi
)
def
insert_entry
(
self
,
collection_name
,
content
:
dict
):
def
insert_entry
(
self
,
collection_name
,
content
:
dict
):
collection
=
self
.
_database
[
collection_name
]
collection
=
self
.
_database
[
collection_name
]
return
collection
.
insert_one
(
content
)
return
collection
.
insert_one
(
content
)
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment