Commit a113a0cc authored by skoulouzis's avatar skoulouzis Committed by GitHub

Merge pull request #68 from skoulouzis/argo-usecase

Argo usecase
parents becf9b9f fa5f5244
......@@ -2,8 +2,7 @@
FROM ubuntu:latest
ARG RABBITMQ_HOST=127.0.0.1
ARG MONOG_HOST=127.0.0.1
# ARG RABBITMQ_HOST=127.0.0.1
ARG RELESE_VERSION=0.1
ARG CERTS=/etc/grid-security/certificates
......@@ -22,6 +21,7 @@ RUN pip install tosca-parser
RUN pip install pika
RUN export LC_ALL="en_US.UTF-8"
RUN pip install paramiko
RUN pip install ansible
......@@ -30,8 +30,8 @@ WORKDIR /root/
RUN wget https://github.com/QCAPI-DRIP/DRIP-integradation/releases/download/$RELESE_VERSION/drip-deployer.tar.gz
RUN tar -xzvf drip-deployer.tar.gz
WORKDIR drip-deployer
ENTRYPOINT python rpc_server.py 172.17.0.2
ENTRYPOINT sleep 10 && python /root/drip-deployer/rpc_server.py $RABBITMQ_HOST
# docker build -t drip-deployer .
# docker run --name drip-deployer-inst -d drip-deployer
......@@ -15,16 +15,51 @@ services:
- "15671:15671"
restart: always
drip-planner:
#drip-planner:
#depends_on:
#- rabbit
#build:
#context: ./planner
#dockerfile: Dockerfile
#image: drip-planner:latest
#restart: always
#environment:
#RABBITMQ_HOST: rabbit
#drip-provisioner:
#depends_on:
#- rabbit
#build:
#context: ./provisioner
#dockerfile: Dockerfile
#image: drip-provisioner:latest
#restart: always
#environment:
#RABBITMQ_HOST: rabbit
drip-deployer:
depends_on:
- rabbit
- rabbit
build:
context: ./planner
context: ./deployer
dockerfile: Dockerfile
image: drip-deployer:latest
restart: always
environment:
- RELESE_VERSION=0.3
image: drip-planner:latest
RABBITMQ_HOST: rabbit
drip-manager:
depends_on:
- rabbit
build:
context: ./manager
dockerfile: Dockerfile
image: drip-manager:latest
restart: always
environment:
RABBITMQ_HOST: rabbit
command: "python rpc_server.py $RABBITMQ_HOST"
\ No newline at end of file
MONOG_HOST: mongo
ports:
- "8080:8080"
#docker-compose up
......@@ -3,8 +3,8 @@ FROM ubuntu:latest
MAINTAINER S. Koulouzis "S.Koulouzis@uva.nl"
ARG RABBITMQ_HOST=127.0.0.1
ARG MONOG_HOST=127.0.0.1
# ARG RABBITMQ_HOST=127.0.0.1
# ARG MONOG_HOST=127.0.0.1
ARG RELESE_VERSION=0.1
ARG CERTS=/etc/grid-security/certificates
......@@ -13,8 +13,8 @@ ARG CERTS=/etc/grid-security/certificates
ENV DEBIAN_FRONTEND=noninteractive
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends
RUN apt-get upgrade -y
RUN apt-get install -y wget git software-properties-common build-essential libssl-dev libffi-dev myproxy
RUN apt-get install -y python-software-properties debconf-utils apt-utils
RUN apt-get install -y wget software-properties-common mongodb-clients
# RUN apt-get install -y python-software-properties debconf-utils apt-utils
RUN add-apt-repository -y ppa:webupd8team/java
RUN apt-get update
RUN "oracle-java8-installer shared/accepted-oracle-license-v1-1 select true" | debconf-set-selections
......@@ -31,23 +31,17 @@ RUN tar -xzf apache-tomcat-7.0.75.tar.gz
WORKDIR apache-tomcat-7.0.75
RUN wget https://github.com/QCAPI-DRIP/DRIP-integradation/releases/download/$RELESE_VERSION/drip-api-1.0-SNAPSHOT.tar.gz
RUN tar -xzvf drip-api-1.0-SNAPSHOT.tar.gz
RUN sed -ie "s/^message.broker.host=.*/message.broker.host=$RABBITMQ_HOST/" drip-api-1.0-SNAPSHOT/WEB-INF/classes/drip.properties
RUN sed -ie "s/^db.host=.*/db.host=$MONOG_HOST/" drip-api-1.0-SNAPSHOT/WEB-INF/classes/drip.properties
RUN cat drip-api-1.0-SNAPSHOT/WEB-INF/classes/drip.properties
RUN mv drip-api-1.0-SNAPSHOT webapps/drip-api
WORKDIR /root
RUN echo "/root/apache-tomcat-7.0.75/bin/startup.sh" >> startAll.sh
RUN echo "tail -f ./apache-tomcat-7.0.75/logs/catalina.out" >> startAll.sh
RUN echo "while true" >> startAll.sh
RUN echo "do" >> startAll.sh
RUN echo "sleep 10s" >> startAll.sh
RUN echo "done" >> startAll.sh
EXPOSE 8080
# ENTRYPOINT sed -ie "s/^message.broker.host=.*/message.broker.host=$RABBITMQ_HOST/" /root/apache-tomcat-7.0.75/webapps/drip-api/WEB-INF/classes/drip.properties && sed -ie "s/^db.host=.*/db.host=$MONOG_HOST/" /root/apache-tomcat-7.0.75/webapps/drip-api/WEB-INF/classes/drip.properties && sleep 10 && /root/apache-tomcat-7.0.75/bin/shutdown.sh && /root/apache-tomcat-7.0.75/bin/startup.sh && tail -f ./apache-tomcat-7.0.75/logs/catalina.out
ENTRYPOINT sed -ie "s/^message.broker.host=.*/message.broker.host=$RABBITMQ_HOST/" /root/apache-tomcat-7.0.75/webapps/drip-api/WEB-INF/classes/drip.properties && sed -ie "s/^db.host=.*/db.host=$MONOG_HOST/" /root/apache-tomcat-7.0.75/webapps/drip-api/WEB-INF/classes/drip.properties && /root/apache-tomcat-7.0.75/bin/shutdown.sh && sleep 20 && mongo -eval 'db.user.insert({"password":"$2a$10$QdysFgsH0sl6Y4BD84UhGO7yyNfoDPXjjEHkDJ3pX6cRfHDj2Q0BO","roles":["ADMIN"],"username":"admin","accountNonExpired":true,"accountNonLocked":true,"credentialsNonExpired":true,"enabled":true})' $MONOG_HOST/drip && /root/apache-tomcat-7.0.75/bin/startup.sh && tail -f /root/apache-tomcat-7.0.75/logs/catalina.out
EXPOSE 8080
RUN chmod +x startAll.sh
ENTRYPOINT ./startAll.sh
# To build: docker build -t drip-manager .
# To run: docker run --name drip-manager-inst -p 127.0.0.1:8080:8080 -d drip-manager
......
......@@ -3,6 +3,8 @@ FROM ubuntu:latest
MAINTAINER S. Koulouzis "S.Koulouzis@uva.nl"
ARG RELESE_VERSION=0.1
ENV DEBIAN_FRONTEND=noninteractive
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends
......@@ -22,12 +24,12 @@ RUN pip install paramiko
# Install deployer
# Install planner
WORKDIR /root/
RUN wget https://github.com/QCAPI-DRIP/DRIP-integradation/releases/download/$RELESE_VERSION/drip-deployer.tar.gz
RUN tar -xzvf drip-deployer.tar.gz
RUN wget https://github.com/QCAPI-DRIP/DRIP-integradation/releases/download/$RELESE_VERSION/drip-planner.tar.gz
RUN tar -xzvf drip-planner.tar.gz
ENTRYPOINT python /root/drip-planner/rpc_server.py $RABBITMQ_HOST
ENTRYPOINT sleep 10 && python /root/drip-planner/rpc_server.py $RABBITMQ_HOST
# docker build -t drip-planner .
# docker run --name drip-planner-inst -d drip-planner
......@@ -4,8 +4,8 @@ FROM ubuntu:latest
MAINTAINER Huan "h.zhou@uva.nl"
ARG RABBITMQ_HOST=127.0.0.1
ARG MONOG_HOST=127.0.0.1
# ARG RABBITMQ_HOST=127.0.0.1
# ARG MONOG_HOST=127.0.0.1
ARG RELESE_VERSION=0.1
ARG CERTS=/etc/grid-security/certificates
......@@ -37,12 +37,11 @@ RUN ./fedcloud-ui.sh
RUN wget https://github.com/QCAPI-DRIP/DRIP-integradation/releases/download/$RELESE_VERSION/drip-provisioner-1.0-SNAPSHOT-jar-with-dependencies.jar
RUN wget https://github.com/QCAPI-DRIP/DRIP-integradation/releases/download/$RELESE_VERSION/database.tar.gz
RUN tar -xzvf database.tar.gz
RUN echo rabbitmq.host=$RABBITMQ_HOST >> provisioner.properties
RUN echo trusted.certificates.folder=$CERTS >> provisioner.properties
RUN echo domain.info.path=/root/drip-provisioner >> provisioner.properties
RUN cat provisioner.properties
ENTRYPOINT java -jar /root/drip-provisioner/drip-provisioner-1.0-SNAPSHOT-jar-with-dependencies.jar /root/drip-provisioner/provisioner.properties
ENTRYPOINT echo rabbitmq.host=$RABBITMQ_HOST >> /root/drip-provisioner/provisioner.propertie && sleep 10 & java -jar /root/drip-provisioner/drip-provisioner-1.0-SNAPSHOT-jar-with-dependencies.jar /root/drip-provisioner/provisioner.propertie
#Build: docker build -t drip-pro .
# Run: docker run --name drip-pro-inst -d drip-pro
......@@ -3,24 +3,32 @@
tasks:
- name: install pre-req
apt: name="{{ item }}" update_cache=yes state=latest
with_items: [python, tcsh, expect, jq , bc, python-pip, python-dev, build-essential, libgeos-dev, nmap]
with_items: [python, tcsh, expect, jq , bc, python-pip, python-dev, build-essential, libgeos-dev, nmap, git, screen, parallel]
become: true
- {name: pip, pip: name=pyproj}
- {name: pip, pip: name=shapely}
- {name: pip, pip: name=psutil}
- {name: pip, pip: name=pyproj,become: yes}
- {name: pip, pip: name=shapely, become: yes}
- {name: pip, pip: name=psutil, become: yes}
- {name: pip pika, pip: name=pika, become: yes}
- {name: pip pika, pip: name=klepto, become: yes}
- {name: mkdir, command: mkfs.ext4 /dev/vdc, become: yes}
- {name: create mount point, file: path=/mnt/data/ state=directory owner=vm_user group=vm_user}
- {name: create mount point, file: path=/mnt/data/source/ state=directory owner=vm_user group=vm_user}
- {name: create mount point, file: path=/mnt/data/source/output state=directory owner=vm_user group=vm_user}
- {name: create mount point data, file: path=/mnt/data/ state=directory owner=vm_user group=vm_user}
- {name: create mount point source, file: path=/mnt/data/source/ state=directory owner=vm_user group=vm_user}
- {name: create mount point output, file: path=/mnt/data/source/output state=directory owner=vm_user group=vm_user}
- {name: mount, command: mount /dev/vdc /mnt/data, become: yes}
- {name: download dataset, get_url: 'url=ftp://ftp.ifremer.fr/ifremer/coriolis/co0547-bigdata-archive/envriplus_euroargo_bigdata.tar.gz dest=/mnt/data'}
- {name: untar, unarchive: src=/mnt/data/envriplus_euroargo_bigdata.tar.gz dest=/mnt/data/}
- {name: download partitioning script, get_url: 'url=https://raw.githubusercontent.com/skoulouzis/dockerfiles/master/ArgoDiffusion/scripts/partitioning.py dest=/mnt/data/source/'}
- {name: download getDelta for time_cover, get_url: 'url=https://raw.githubusercontent.com/skoulouzis/dockerfiles/master/ArgoDiffusion/scripts/getDelta.py dest=/mnt/data/source/'}
- {name: download ar_bigmetadata for the filter script, get_url: 'url=https://raw.githubusercontent.com/skoulouzis/dockerfiles/master/ArgoDiffusion/scripts/ar_bigmetadata.json dest=/mnt/data/source/'}
- {name: download for filter script, get_url: 'url=https://raw.githubusercontent.com/skoulouzis/dockerfiles/master/ArgoDiffusion/scripts/configuration.json dest=/mnt/data/source/'}
- {name: exec partitioning, command: python /mnt/data/source/partitioning.py /mnt/data/source/configuration.json, become_user: vm_user}
- {name: create data_argo.txt, file: path=//mnt/data/data_argo.txt state=touch owner=vm_user group=vm_user}
- {name: untar, unarchive: src=/mnt/data/envriplus_euroargo_bigdata.tar.gz dest=/mnt/data/ remote_src=true}
- {name: create workspace, file: path=/home/vm_user/workspace state=directory owner=vm_user group=vm_user}
- {name: clone repo, git: 'repo=https://github.com/skoulouzis/dockerfiles.git dest=/home/vm_user/workspace/dockerfiles version=HEAD force=yes'}
- {name: conf script, command: cp /home/vm_user/workspace/dockerfiles/ArgoDiffusion/scripts/ar_bigmetadata.json /mnt/data/source/ar_bigmetadata.json, become: no}
- {name: conf ssh cp, command: cp /etc/ssh/sshd_config /tmp, become: yes}
- {name: conf ssh chown vm_user, command: chown vm_user /tmp/sshd_config, become: yes}
- {name: conf ssh echo, command: echo "" >> sshd_config && echo "UseDNS no" >> sshd_config, become: no}
- {name: conf ssh chown root, command: chown root /tmp/sshd_config, become: yes}
- {name: conf ssh mv, command: mv /tmp/sshd_config /etc/ssh/, become: yes}
- {name: conf ssh restart, command: service ssh restart, become: yes}
- {name: screen conf, command: echo "logfile /mnt/data/argoBenchmark" > /home/vm_user/.screenrc, become_user: vm_user}
\ No newline at end of file
......@@ -26,13 +26,11 @@ import java.util.Map;
import java.util.concurrent.TimeoutException;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.regex.Matcher;
import nl.uva.sne.drip.api.dao.DeployDao;
import nl.uva.sne.drip.api.exception.NotFoundException;
import nl.uva.sne.drip.api.rpc.DRIPCaller;
import nl.uva.sne.drip.api.rpc.DeployerCaller;
import nl.uva.sne.drip.api.v1.rest.DeployController;
import nl.uva.sne.drip.drip.commons.data.v1.external.CloudCredentials;
import nl.uva.sne.drip.drip.commons.data.v1.external.DeployRequest;
import nl.uva.sne.drip.drip.commons.data.v1.external.DeployParameter;
import nl.uva.sne.drip.drip.commons.data.v1.external.DeployResponse;
......@@ -271,7 +269,7 @@ public class DeployService {
Map<String, String> nodeTypeCache = new HashMap<>();
Map<String, String> domainCache = new HashMap<>();
Map<String, String> osTypeCache = new HashMap<>();
// Map<String, String> cloudProviderCache = new HashMap<>();
Map<String, String> cloudProviderCache = new HashMap<>();
for (AnsibleOutput ansOut : outputList) {
Map<String, Object> map = provisionService.findOne(deployInfo.getProvisionID()).getKeyValue();
......@@ -316,6 +314,7 @@ public class DeployService {
}
}
os = (String) component.get("OStype");
nodeTypeCache.put(ansOut.getHost(), nodeType);
domainCache.put(ansOut.getHost(), domain);
osTypeCache.put(ansOut.getHost(), os);
......@@ -326,7 +325,8 @@ public class DeployService {
ansOut.setVmType(nodeType);
ansOut.setCloudDeploymentDomain(domain);
ansOut.setProvisionID(deployInfo.getProvisionID());
ansOut.setCloudProviderName(os);
// ansOut.setCloudProvider(provider);
ansOut = ansibleOutputService.save(ansOut);
BenchmarkResult benchmarkResult = parseSaveBenchmarkResult(ansOut);
......
......@@ -105,11 +105,6 @@ public class AnsibleOutput extends OwnedObject {
public String getProvisionID() {
return provisionID;
}
public void setCloudProviderName(String cloudProvider) {
this.setCloudProvider(cloudProvider);
}
/**
* @return the cloudProvider
*/
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment