Merge branch 'feature/ci-cd-setup' into develop

README.md

@@ -2,18 +2,18 @@
 We propose a framework underlying decentralized social media called SMART, capable of finding relevant interest communities without violating users’ privacy and anonymity. Its objective is to improve trust and eliminate malicious actors in participatory exchanges and collaborative decision making.
 
 ## Project structure
+The *tools* folder contains supportive scripts used during development and must not be used by any microservice.
+
+### Source code in *src/*
 The five folders *data-hub*, *message-broker*, *rest-gateway*, *transaction-hub-in* and *transaction-hub-out* contain the microservices for the five architecture layers respectively.
 
 The *modules* folder contains various Python modules used by multiple microservices to improve code-reuse. When building the Docker images this folder is copied into the app path.
 
-The *tools* folder contains supportive scripts used during development and must not be used by any microservice.
-
-The scripts *build.py* and *deploy.py* are used to create Docker images and deploy these images in Kubernetes respectively.
-
-The *images* folder holds the images used in this document.
+### Deployment scripts in *bin/*
+The scripts *build.py* and *deploy.py* are used to create Docker images and deploy these images with Kubernetes respectively.
 
 ## Overall microservice architecture
-![SMART architecture image](images/smart-architecture.png)
+![SMART architecture image](documentation/images/smart-architecture.png)
 
 ## Trace input handling
 1. New traces are POSTed to the REST gateway
@@ -21,4 +21,4 @@ The *images* folder holds the images used in this document.
 1. The trace retrieval microservice receives the message, stores the trace in its document-based database and sends a notification to the message broker
 1. The semantic linking microservice receives the notification and GETs all traces (including the new one) from the trace retrieval microservice
 1. All traces can now be processed
-![Input handling image](images/input-handling.png)
+![Input handling image](documentation/images/input-handling.png)

bin/build.py

+import os
+import shutil
+import sys
+
+if len(sys.argv) != 2:
+    raise Exception("Push to Docker Hub will not work, please provide username as argument")
+
+DOCKER_COMPOSE_NAME = "Dockerfile"
+ROOT = './'
+SOURCEPATH = f'{ROOT}src/'
+DOCKER_USERNAME = sys.argv[1]
+
+paths = []
+for r, _, f in os.walk(SOURCEPATH):
+    for filename in f:
+        if DOCKER_COMPOSE_NAME == filename:
+            paths.append(os.path.normpath(r))
+
+command_args = [{'path': path,
+                 'name': str(path).split(os.path.normpath('/'))[-1]}
+                for path
+                in paths]
+
+res_str = []
+for command_arg in command_args:
+    path = command_arg['path']
+    image_name = f"{DOCKER_USERNAME}/{command_arg['name']}"
+
+    # copy Dockerfile into root dir to have access to modules folder
+    shutil.copy2(os.path.join(path, DOCKER_COMPOSE_NAME), ROOT)
+
+    # build then remove Dockerfile
+    exit_val = os.system(f"docker image build -t {image_name} {ROOT}")
+    os.remove(os.path.join(ROOT, DOCKER_COMPOSE_NAME))
+
+    res_str.append(f"### {image_name} built with exit code {exit_val}")
+
+    os.system(f"docker push {image_name}")
+
+print(f"Found {len(command_args)} images")
+for s in res_str:
+    print(s)

bin/deploy.py

+import os
+import sys
+
+EXTERNAL_IP = "143.205.173.225"
+
+# apply or delete config
+kube_command = 'apply'
+if len(sys.argv) == 2:
+    kube_command = sys.argv[1]
+
+paths = []
+for p, _, f in os.walk('./'):
+    for file in f:
+        if 'deployment.yml' == file:
+            paths.append(os.path.normpath(p))
+
+for path in paths:
+    os.system(f"kubectl {kube_command} -f {path}")
+    
+    # if kube_command == 'apply':
+    #     # apply external ip
+    #     service_name = path.split(os.path.normpath('/'))[-2] # -1 is deployment folder
+    #     if service_name.split('-')[-1] == 'microservice':
+    #         # remove microservice suffix
+    #         service_name = service_name[:len(service_name)-len('-microservice')]
+        
+    #     os.system(f"kubectl patch service {service_name} -p '{{\"spec\":{{\"externalIPs\":[\"{EXTERNAL_IP}\"]}}}}'")
\ No newline at end of file

build.py

-import os
-import shutil
-
-docker_reg = 'localhost:5000'
-DOCKER_COMPOSE_NAME = "Dockerfile"
-
-paths = []
-for r, _, f in os.walk('./'):
-    for file in f:
-        if DOCKER_COMPOSE_NAME == file:
-            paths.append(os.path.join(r, ''))
-
-command_args = [{'path': path,
-    'name': str(path).split('/')[-2]}
-    for path
-    in paths]
-
-for command_arg in command_args:
-    path = command_arg['path']
-    image_name = f"{docker_reg}/{command_arg['name']}"
-
-    # copy Dockerfile into root dir to have access to modules folder
-    shutil.copy2(os.path.join(path, DOCKER_COMPOSE_NAME), './')
-    os.system(f"docker image build -t {image_name} .")
-    os.remove(DOCKER_COMPOSE_NAME)
-
-    os.system(f"docker push {image_name}")
-
-print(f"Found {len(command_args)} images")
\ No newline at end of file

data-hub/reputation-calculation-microservice/Dockerfile

-FROM python:3
-LABEL maintainer="Alexander Lercher"
-
-ENV http_proxy http://proxy.uni-klu.ac.at:3128/
-ENV https_proxy http://proxy.uni-klu.ac.at:3128/
-
-RUN apt-get update
-RUN pip install flask
-RUN pip install connexion[swagger-ui]
-
-EXPOSE 5000
-
-WORKDIR /app
-COPY data-hub/reputation-calculation-microservice/app/ /app/
-RUN chmod a+x main.py
-
-CMD ["python", "./main.py"]
\ No newline at end of file

data-hub/stage-discovery-microservice/Dockerfile

-FROM python:3
-LABEL maintainer="Alexander Lercher"
-
-ENV http_proxy http://proxy.uni-klu.ac.at:3128/
-ENV https_proxy http://proxy.uni-klu.ac.at:3128/
-
-RUN apt-get update
-RUN pip install flask
-RUN pip install connexion[swagger-ui]
-
-EXPOSE 5000
-
-WORKDIR /app
-COPY data-hub/stage-discovery-microservice/app/ /app/
-RUN chmod a+x main.py
-
-CMD ["python", "./main.py"]
\ No newline at end of file

deploy.py

-import os
-import sys
-
-kube_command = 'apply'
-if len(sys.argv) > 1:
-    kube_command = sys.argv[1]
-
-paths = []
-for p, _, f in os.walk('./'):
-    for file in f:
-        if 'deployment.yml' == file:
-            paths.append(os.path.join(p, ''))
-
-for path in paths:
-    os.system(f"kubectl {kube_command} -f {path}")
\ No newline at end of file

data-hub/community-detection-microservice/Dockerfile → src/data-hub/agent-discovery-microservice/Dockerfile

@@ -11,7 +11,7 @@ RUN pip install connexion[swagger-ui]
 EXPOSE 5000
 
 WORKDIR /app
-COPY data-hub/community-detection-microservice/app/ /app/
+COPY src/data-hub/agent-discovery-microservice/app/ /app/
 RUN chmod a+x main.py
 
 CMD ["python", "./main.py"]
\ No newline at end of file

data-hub/agent-discovery-microservice/deployment/deployment.yml → src/data-hub/agent-discovery-microservice/deployment/deployment.yml

@@ -4,8 +4,6 @@ metadata:
   name: agent-discovery
 spec:
   type: LoadBalancer
-  externalIPs:
-    - 143.205.173.102
   selector:
       app: agent-discovery
   ports:
@@ -31,6 +29,6 @@ spec:
     spec:
       containers:
       - name: agent-discovery
-        image: 143.205.173.97:5000/agent-discovery-microservice
+        image: alexx882/agent-discovery-microservice
         ports:
         - containerPort: 5000
\ No newline at end of file

data-hub/democratic-reasoning-microservice/Dockerfile → src/data-hub/community-detection-microservice/Dockerfile

@@ -11,7 +11,7 @@ RUN pip install connexion[swagger-ui]
 EXPOSE 5000
 
 WORKDIR /app
-COPY data-hub/democratic-reasoning-microservice/app/ /app/
+COPY src/data-hub/community-detection-microservice/app/ /app/
 RUN chmod a+x main.py
 
 CMD ["python", "./main.py"]
\ No newline at end of file

data-hub/community-detection-microservice/deployment/deployment.yml → src/data-hub/community-detection-microservice/deployment/deployment.yml

@@ -4,8 +4,6 @@ metadata:
   name: community-detection
 spec:
   type: LoadBalancer
-  externalIPs:
-    - 143.205.173.102
   selector:
       app: community-detection
   ports:
@@ -31,6 +29,6 @@ spec:
     spec:
       containers:
       - name: community-detection
-        image: 143.205.173.97:5000/community-detection-microservice
+        image: alexx882/community-detection-microservice
         ports:
         - containerPort: 5000
\ No newline at end of file

src/data-hub/democratic-reasoning-microservice/Dockerfile

+FROM python:3
+LABEL maintainer="Alexander Lercher"
+
+ENV http_proxy http://proxy.uni-klu.ac.at:3128/
+ENV https_proxy http://proxy.uni-klu.ac.at:3128/
+
+RUN apt-get update
+RUN pip install flask
+RUN pip install connexion[swagger-ui]
+
+EXPOSE 5000
+
+WORKDIR /app
+COPY src/data-hub/democratic-reasoning-microservice/app/ /app/
+RUN chmod a+x main.py
+
+CMD ["python", "./main.py"]
\ No newline at end of file

data-hub/democratic-reasoning-microservice/deployment/deployment.yml → src/data-hub/democratic-reasoning-microservice/deployment/deployment.yml

@@ -4,8 +4,6 @@ metadata:
   name: democratic-reasoning
 spec:
   type: LoadBalancer
-  externalIPs:
-    - 143.205.173.102
   selector:
       app: democratic-reasoning
   ports:
@@ -31,6 +29,6 @@ spec:
     spec:
       containers:
       - name: democratic-reasoning
-        image: 143.205.173.97:5000/democratic-reasoning-microservice
+        image: alexx882/democratic-reasoning-microservice
         ports:
         - containerPort: 5000
\ No newline at end of file

data-hub/role-discovery-microservice/Dockerfile → src/data-hub/geo-profiling-microservice/Dockerfile

@@ -11,7 +11,7 @@ RUN pip install connexion[swagger-ui]
 EXPOSE 5000
 
 WORKDIR /app
-COPY data-hub/role-discovery-microservice/app/ /app/
+COPY src/data-hub/geo-profiling-microservice/app/ /app/
 RUN chmod a+x main.py
 
 CMD ["python", "./main.py"]
\ No newline at end of file

data-hub/geo-profiling-microservice/deployment/deployment.yml → src/data-hub/geo-profiling-microservice/deployment/deployment.yml

@@ -4,8 +4,6 @@ metadata:
   name: geo-profiling
 spec:
   type: LoadBalancer
-  externalIPs:
-    - 143.205.173.102
   selector:
       app: geo-profiling
   ports:
@@ -31,6 +29,6 @@ spec:
     spec:
       containers:
       - name: geo-profiling
-        image: 143.205.173.97:5000/geo-profiling-microservice
+        image: alexx882/geo-profiling-microservice
         ports:
         - containerPort: 5000
\ No newline at end of file

data-hub/pareto-trust-microservice/Dockerfile → src/data-hub/pareto-trust-microservice/Dockerfile

@@ -11,7 +11,7 @@ RUN pip install connexion[swagger-ui]
 EXPOSE 5000
 
 WORKDIR /app
-COPY data-hub/pareto-trust-microservice/app/ /app/
+COPY src/data-hub/pareto-trust-microservice/app/ /app/
 RUN chmod a+x main.py
 
 CMD ["python", "./main.py"]
\ No newline at end of file

data-hub/pareto-trust-microservice/deployment/deployment.yml → src/data-hub/pareto-trust-microservice/deployment/deployment.yml

@@ -4,8 +4,6 @@ metadata:
   name: pareto-trust
 spec:
   type: LoadBalancer
-  externalIPs:
-    - 143.205.173.102
   selector:
       app: pareto-trust
   ports:
@@ -31,6 +29,6 @@ spec:
     spec:
       containers:
       - name: pareto-trust
-        image: 143.205.173.97:5000/pareto-trust-microservice
+        image: alexx882/pareto-trust-microservice
         ports:
         - containerPort: 5000
\ No newline at end of file

transaction-hub-out/event-detection-microservice/Dockerfile → src/data-hub/reputation-calculation-microservice/Dockerfile

@@ -11,7 +11,7 @@ RUN pip install connexion[swagger-ui]
 EXPOSE 5000
 
 WORKDIR /app
-COPY transaction-hub-out/event-detection-microservice/app/ /app/
+COPY src/data-hub/reputation-calculation-microservice/app/ /app/
 RUN chmod a+x main.py
 
 CMD ["python", "./main.py"]
\ No newline at end of file

data-hub/reputation-calculation-microservice/deployment/deployment.yml → src/data-hub/reputation-calculation-microservice/deployment/deployment.yml

@@ -4,8 +4,6 @@ metadata:
   name: reputation-calculation
 spec:
   type: LoadBalancer
-  externalIPs:
-    - 143.205.173.102
   selector:
       app: reputation-calculation
   ports:
@@ -31,6 +29,6 @@ spec:
     spec:
       containers:
       - name: reputation-calculation
-        image: 143.205.173.97:5000/reputation-calculation-microservice
+        image: alexx882/reputation-calculation-microservice
         ports:
         - containerPort: 5000
\ No newline at end of file

data-hub/agent-discovery-microservice/Dockerfile → src/data-hub/role-discovery-microservice/Dockerfile

@@ -11,7 +11,7 @@ RUN pip install connexion[swagger-ui]
 EXPOSE 5000
 
 WORKDIR /app
-COPY data-hub/agent-discovery-microservice/app/ /app/
+COPY src/data-hub/role-discovery-microservice/app/ /app/
 RUN chmod a+x main.py
 
 CMD ["python", "./main.py"]
\ No newline at end of file

data-hub/role-discovery-microservice/deployment/deployment.yml → src/data-hub/role-discovery-microservice/deployment/deployment.yml

@@ -4,8 +4,6 @@ metadata:
   name: role-discovery
 spec:
   type: LoadBalancer
-  externalIPs:
-    - 143.205.173.102
   selector:
       app: role-discovery
   ports:
@@ -31,6 +29,6 @@ spec:
     spec:
       containers:
       - name: role-discovery
-        image: 143.205.173.97:5000/role-discovery-microservice
+        image: alexx882/role-discovery-microservice
         ports:
         - containerPort: 5000
\ No newline at end of file

data-hub/semantic-linking-microservice/Dockerfile → src/data-hub/semantic-linking-microservice/Dockerfile

@@ -12,8 +12,8 @@ RUN pip install pika
 EXPOSE 5000
 
 WORKDIR /app
-COPY data-hub/semantic-linking-microservice/app/ /app/
-COPY modules/ /app/
+COPY src/data-hub/semantic-linking-microservice/app/ /app/
+COPY src/modules/ /app/
 RUN chmod a+x main.py
 
 CMD ["python", "./main.py"]
\ No newline at end of file

data-hub/semantic-linking-microservice/deployment/deployment.yml → src/data-hub/semantic-linking-microservice/deployment/deployment.yml

@@ -4,8 +4,6 @@ metadata:
   name: semantic-linking
 spec:
   type: LoadBalancer
-  externalIPs:
-    - 143.205.173.102
   selector:
       app: semantic-linking
   ports:
@@ -31,6 +29,6 @@ spec:
     spec:
       containers:
       - name: semantic-linking
-        image: 143.205.173.97:5000/semantic-linking-microservice
+        image: alexx882/semantic-linking-microservice
         ports:
         - containerPort: 5000
\ No newline at end of file

data-hub/geo-profiling-microservice/Dockerfile → src/data-hub/stage-discovery-microservice/Dockerfile

@@ -11,7 +11,7 @@ RUN pip install connexion[swagger-ui]
 EXPOSE 5000
 
 WORKDIR /app
-COPY data-hub/geo-profiling-microservice/app/ /app/
+COPY src/data-hub/stage-discovery-microservice/app/ /app/
 RUN chmod a+x main.py
 
 CMD ["python", "./main.py"]
\ No newline at end of file

data-hub/stage-discovery-microservice/deployment/deployment.yml → src/data-hub/stage-discovery-microservice/deployment/deployment.yml

@@ -4,8 +4,6 @@ metadata:
   name: stage-discovery
 spec:
   type: LoadBalancer
-  externalIPs:
-    - 143.205.173.102
   selector:
       app: stage-discovery
   ports:
@@ -31,6 +29,6 @@ spec:
     spec:
       containers:
       - name: stage-discovery
-        image: 143.205.173.97:5000/stage-discovery-microservice
+        image: alexx882/stage-discovery-microservice
         ports:
         - containerPort: 5000
\ No newline at end of file

message-broker/deployment/deployment.yml → src/message-broker/deployment/deployment.yml

@@ -4,8 +4,6 @@ metadata:
   name: rabbit-mq
 spec:
   type: LoadBalancer
-  externalIPs:
-    - 143.205.173.102
   selector:
       app: rabbit-mq
   ports:

src/modules/README.md

+# Shared modules
\ No newline at end of file

rest-gateway/Dockerfile → src/rest-gateway/Dockerfile

@@ -13,8 +13,8 @@ RUN pip install deprecated
 EXPOSE 5000
 
 WORKDIR /app
-COPY rest-gateway/app/ /app/
-COPY modules/ /app/
+COPY src/rest-gateway/app/ /app/
+COPY src/modules/ /app/
 RUN chmod a+x main.py
 
 CMD ["python", "./main.py"]
\ No newline at end of file

rest-gateway/deployment/deployment.yml → src/rest-gateway/deployment/deployment.yml

@@ -4,8 +4,6 @@ metadata:
   name: rest-gateway
 spec:
   type: LoadBalancer
-  externalIPs:
-    - 143.205.173.102
   selector:
       app: rest-gateway
   ports:
@@ -31,6 +29,6 @@ spec:
     spec:
       containers:
       - name: rest-gateway
-        image: 143.205.173.97:5000/rest-gateway
+        image: alexx882/rest-gateway
         ports:
         - containerPort: 5000
\ No newline at end of file

src/transaction-hub-in/network-metrics-retrieval-microservice/Dockerfile

+FROM python:3
+LABEL maintainer="Alexander Lercher"
+
+ENV http_proxy http://proxy.uni-klu.ac.at:3128/
+ENV https_proxy http://proxy.uni-klu.ac.at:3128/
+
+RUN apt-get update
+RUN pip install flask
+RUN pip install connexion[swagger-ui]
+
+EXPOSE 5000
+
+WORKDIR /app
+COPY src/transaction-hub-in/network-metrics-retrieval-microservice/app/ /app/
+RUN chmod a+x main.py
+
+CMD ["python", "./main.py"]
\ No newline at end of file

transaction-hub-in/network-metrics-retrieval-microservice/deployment/deployment.yml → src/transaction-hub-in/network-metrics-retrieval-microservice/deployment/deployment.yml

@@ -4,8 +4,6 @@ metadata:
   name: network-metrics-retrieval
 spec:
   type: LoadBalancer
-  externalIPs:
-    - 143.205.173.102
   selector:
       app: network-metrics-retrieval
   ports:
@@ -31,6 +29,6 @@ spec:
     spec:
       containers:
       - name: network-metrics-retrieval
-        image: 143.205.173.97:5000/network-metrics-retrieval-microservice
+        image: alexx882/network-metrics-retrieval-microservice
         ports:
         - containerPort: 5000
\ No newline at end of file