Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
S
SMART
Project
Project
Details
Activity
Releases
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
3
Issues
3
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Registry
Registry
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
UNI-KLU
SMART
Commits
fec1e504
Commit
fec1e504
authored
Oct 22, 2020
by
Alexander Lercher
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Uploading data with updated user demand
parent
e359a787
Changes
4
Hide whitespace changes
Inline
Side-by-side
Showing
4 changed files
with
77 additions
and
13 deletions
+77
-13
convert_layers.py
...b/role-stage-discovery-microservice/app/convert_layers.py
+46
-0
repository.py
...ub/role-stage-discovery-microservice/app/db/repository.py
+4
-1
dummy_upload.py
...ata-hub/semantic-linking-microservice/app/dummy_upload.py
+8
-7
add_smart_energy.py
..._use_case_scripts/smart-energy/tables/add_smart_energy.py
+19
-5
No files found.
src/data-hub/role-stage-discovery-microservice/app/convert_layers.py
0 → 100644
View file @
fec1e504
import
sys
import
os
modules_path
=
'../../../modules/'
if
os
.
path
.
exists
(
modules_path
):
sys
.
path
.
insert
(
1
,
modules_path
)
import
json
from
db.entities
import
Cluster
from
typing
import
List
from
db.repository
import
Repository
repo
=
Repository
()
def
get_clusters
(
layer_file
)
->
List
[
Cluster
]:
with
open
(
layer_file
,
'r'
)
as
file
:
clusters
=
json
.
loads
(
file
.
read
())
return
[
Cluster
(
cluster_dict
=
cluster
,
from_db
=
True
)
for
cluster
in
clusters
]
def
store_generic_clusters
(
clusters
:
List
[
Cluster
],
layer
):
print
([
c
.
label
for
c
in
clusters
][
0
:
10
])
return
try
:
with
open
(
f
'{layer}.json'
,
'w'
)
as
file
:
cluster_dicts
=
[
c
.
to_serializable_dict
(
for_db
=
False
)
for
c
in
clusters
]
file
.
write
(
json
.
dumps
(
cluster_dicts
))
except
:
print
(
f
"failed writing {layer}"
)
try
:
for
cluster
in
clusters
:
repo
.
add_cluster
(
cluster
)
except
:
print
(
f
"failed uploading {layer}"
)
layers
=
[
'Position_Layer.json'
,
'Solar_Production_Layer.json'
]
for
layer
in
layers
:
clusts
:
List
[
Cluster
]
=
get_clusters
(
layer
)
# print(len(clusts))
store_generic_clusters
(
clusts
,
layer
)
\ No newline at end of file
src/data-hub/role-stage-discovery-microservice/app/db/repository.py
View file @
fec1e504
...
...
@@ -34,7 +34,7 @@ class Repository(MongoRepositoryBase):
def
get_layers
(
self
)
->
List
[
Layer
]:
'''Retrieves all layers from the db, independent of use-case.'''
entries
=
super
()
.
get_entries
(
self
.
_layer_collection
)
entries
=
super
()
.
get_entries
(
self
.
_layer_collection
,
projection
=
{
'_id'
:
0
}
)
return
[
Layer
(
e
)
for
e
in
entries
]
def
get_layers_for_use_case
(
self
,
use_case
:
str
)
->
Layer
:
...
...
@@ -81,6 +81,9 @@ class Repository(MongoRepositoryBase):
#endregion
#region Clusters
def
add_cluster
(
self
,
cluster
:
Cluster
):
super
()
.
insert_entry
(
self
.
_clusters_collection
,
cluster
.
to_serializable_dict
(
for_db
=
True
))
def
add_clusters
(
self
,
clusters
:
List
[
Cluster
]):
cluster_dicts
=
[
c
.
to_serializable_dict
(
for_db
=
True
)
for
c
in
clusters
]
super
()
.
insert_many
(
self
.
_clusters_collection
,
cluster_dicts
)
...
...
src/data-hub/semantic-linking-microservice/app/dummy_upload.py
View file @
fec1e504
...
...
@@ -22,10 +22,10 @@ def upload_transaction(transaction):
uid
=
hashlib
.
sha256
(
f
"{transaction['Customer']}{transaction['Timestamp']}"
.
encode
(
"utf-8"
))
.
hexdigest
()
transaction
[
'UniqueID'
]
=
uid
t
=
{
'use_case'
:
'smart-energy'
,
'table'
:
'smart-energy'
,
'use_case'
:
transaction
[
'ApplicationType'
]
,
'table'
:
transaction
[
'docType'
]
,
'id'
:
uid
,
'properties'
:
transaction
'properties'
:
transaction
,
}
handler
.
handle_new_trace
(
t
)
...
...
@@ -40,15 +40,16 @@ if __name__ == '__main__':
for
row
in
reader
:
transaction
=
{}
transaction
[
'ApplicationType'
]
=
'smart-energy'
transaction
[
'docType'
]
=
'smart-energy'
transaction
[
'docType'
]
=
'smart-energy
-paper
'
for
idx
in
range
(
len
(
row
)):
transaction
[
titles
[
idx
]]
=
row
[
idx
]
if
int
(
transaction
[
'Customer'
])
<
20
:
continue
# also include the user demand, as Total_Demand_WMh is not per user
energy_cons
=
float
(
transaction
[
'Energy_Consumption_kWh'
])
if
transaction
[
'Energy_Consumption_kWh'
]
is
not
None
and
transaction
[
'Energy_Consumption_kWh'
]
!=
""
else
0
heating_cons
=
float
(
transaction
[
'Heating_Consumption_kWh'
])
if
transaction
[
'Heating_Consumption_kWh'
]
is
not
None
and
transaction
[
'Heating_Consumption_kWh'
]
!=
""
else
0
transaction
[
'User_Demand_kWh'
]
=
heating_cons
+
energy_cons
if
transaction
[
'Customer'
]
!=
old_c
:
# only upload until 200 for now
old_c
=
transaction
[
'Customer'
]
print
(
f
"uploading for {old_c}"
)
...
...
src/participation-hub/business-logic-microservice/app/_add_use_case_scripts/smart-energy/tables/add_smart_energy.py
View file @
fec1e504
...
...
@@ -20,7 +20,8 @@ def add_table(use_case: str, table_name: str):
"Price_AUD/MWh"
,
"Total_Demand_MWh"
,
"Latitude"
,
"Longitude"
"Longitude"
,
"User_Demand_kWh"
,
]
columns
=
{
c
.
replace
(
"/"
,
"_"
)
:
c
for
c
in
columns
}
...
...
@@ -119,7 +120,7 @@ def add_layers(use_case:str, table_name: str):
},{
"use_case"
:
use_case
,
"table"
:
table_name
,
"name"
:
"
Posi
tion_Layer"
,
"name"
:
"
Loca
tion_Layer"
,
"properties"
:
[
"UniqueID"
,
"Customer"
,
...
...
@@ -133,6 +134,21 @@ def add_layers(use_case:str, table_name: str):
"Longitude"
,
]
},
{
"use_case"
:
use_case
,
"table"
:
table_name
,
"name"
:
"User_Demand_Layer"
,
"properties"
:
[
"UniqueID"
,
"Customer"
,
"Postcode"
,
"Timestamp"
,
"User_Demand_kWh"
],
"cluster_properties"
:
[
"User_Demand_kWh"
,
]
},
]
jwt
=
TokenManager
.
getInstance
()
.
getToken
()
...
...
@@ -150,10 +166,8 @@ def add_layers(use_case:str, table_name: str):
print
(
url
+
": "
+
str
(
response
.
status_code
))
def
main
(
use_case
:
str
):
def
main
(
use_case
:
str
=
"smart-energy"
,
table_name
:
str
=
"smart-energy-paper"
):
print
(
"SMART-ENERGY"
)
table_name
=
"smart-energy"
add_table
(
use_case
,
table_name
)
add_layers
(
use_case
,
table_name
)
\ No newline at end of file
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment