Merge pull request #68 from skoulouzis/argo-usecase

Argo usecase

Dockerfiles/deployer/Dockerfile

@@ -2,8 +2,7 @@
 FROM ubuntu:latest
 
 
-ARG RABBITMQ_HOST=127.0.0.1
-ARG MONOG_HOST=127.0.0.1
+# ARG RABBITMQ_HOST=127.0.0.1
 ARG RELESE_VERSION=0.1
 ARG CERTS=/etc/grid-security/certificates
 
@@ -22,6 +21,7 @@ RUN pip install tosca-parser
 RUN pip install pika
 RUN export LC_ALL="en_US.UTF-8"
 RUN pip install paramiko
+RUN pip install ansible
 
 
 
@@ -30,8 +30,8 @@ WORKDIR /root/
 RUN wget https://github.com/QCAPI-DRIP/DRIP-integradation/releases/download/$RELESE_VERSION/drip-deployer.tar.gz
 RUN tar -xzvf drip-deployer.tar.gz
 
-WORKDIR drip-deployer
-ENTRYPOINT python rpc_server.py 172.17.0.2
+
+ENTRYPOINT sleep 10 && python /root/drip-deployer/rpc_server.py $RABBITMQ_HOST
 
 # docker build -t drip-deployer .
 # docker run --name drip-deployer-inst -d drip-deployer

Dockerfiles/docker-compose.yml

@@ -15,16 +15,51 @@ services:
             - "15671:15671"
         restart: always
     
-    drip-planner:
+    #drip-planner:
+        #depends_on:
+            #- rabbit
+        #build:
+            #context: ./planner
+            #dockerfile: Dockerfile
+        #image: drip-planner:latest
+        #restart: always
+        #environment:
+            #RABBITMQ_HOST: rabbit
+            
+    #drip-provisioner:
+        #depends_on:
+            #- rabbit
+        #build:
+            #context: ./provisioner
+            #dockerfile: Dockerfile
+        #image: drip-provisioner:latest
+        #restart: always
+        #environment:
+            #RABBITMQ_HOST: rabbit      
+            
+    drip-deployer:
         depends_on:
-            - rabbit        
+            - rabbit
         build:
-            context: ./planner
+            context: ./deployer
             dockerfile: Dockerfile
+        image: drip-deployer:latest
+        restart: always
         environment:
-            - RELESE_VERSION=0.3
-        image: drip-planner:latest
+            RABBITMQ_HOST: rabbit 
+            
+    drip-manager:
+        depends_on:
+            - rabbit
+        build:
+            context: ./manager
+            dockerfile: Dockerfile
+        image: drip-manager:latest
         restart: always
         environment:
             RABBITMQ_HOST: rabbit
-        command: "python rpc_server.py $RABBITMQ_HOST"
\ No newline at end of file
+            MONOG_HOST: mongo
+        ports:
+            - "8080:8080"        
+            
+#docker-compose up

Dockerfiles/manager/Dockerfile

@@ -3,8 +3,8 @@ FROM ubuntu:latest
 MAINTAINER S. Koulouzis "S.Koulouzis@uva.nl"
 
 
-ARG RABBITMQ_HOST=127.0.0.1
-ARG MONOG_HOST=127.0.0.1
+# ARG RABBITMQ_HOST=127.0.0.1
+# ARG MONOG_HOST=127.0.0.1
 ARG RELESE_VERSION=0.1
 ARG CERTS=/etc/grid-security/certificates
 
@@ -13,8 +13,8 @@ ARG CERTS=/etc/grid-security/certificates
 ENV DEBIAN_FRONTEND=noninteractive 
 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends 
 RUN apt-get upgrade -y
-RUN apt-get install -y wget git software-properties-common build-essential libssl-dev libffi-dev myproxy 
-RUN apt-get install -y python-software-properties debconf-utils apt-utils
+RUN apt-get install -y wget software-properties-common mongodb-clients 
+# RUN apt-get install -y python-software-properties debconf-utils apt-utils
 RUN add-apt-repository -y ppa:webupd8team/java
 RUN apt-get update
 RUN "oracle-java8-installer shared/accepted-oracle-license-v1-1 select true" | debconf-set-selections
@@ -31,23 +31,17 @@ RUN tar -xzf apache-tomcat-7.0.75.tar.gz
 WORKDIR apache-tomcat-7.0.75
 RUN wget https://github.com/QCAPI-DRIP/DRIP-integradation/releases/download/$RELESE_VERSION/drip-api-1.0-SNAPSHOT.tar.gz 
 RUN tar -xzvf drip-api-1.0-SNAPSHOT.tar.gz
-RUN sed -ie "s/^message.broker.host=.*/message.broker.host=$RABBITMQ_HOST/" drip-api-1.0-SNAPSHOT/WEB-INF/classes/drip.properties
-RUN sed -ie "s/^db.host=.*/db.host=$MONOG_HOST/" drip-api-1.0-SNAPSHOT/WEB-INF/classes/drip.properties
 RUN cat drip-api-1.0-SNAPSHOT/WEB-INF/classes/drip.properties
 RUN mv drip-api-1.0-SNAPSHOT webapps/drip-api
 
 WORKDIR /root
-RUN echo "/root/apache-tomcat-7.0.75/bin/startup.sh" >> startAll.sh
-RUN echo "tail -f ./apache-tomcat-7.0.75/logs/catalina.out" >> startAll.sh
-RUN echo "while true" >> startAll.sh
-RUN echo "do" >> startAll.sh
-RUN echo "sleep 10s" >> startAll.sh
-RUN echo "done" >> startAll.sh
+EXPOSE 8080
 
+# ENTRYPOINT sed -ie "s/^message.broker.host=.*/message.broker.host=$RABBITMQ_HOST/" /root/apache-tomcat-7.0.75/webapps/drip-api/WEB-INF/classes/drip.properties && sed -ie "s/^db.host=.*/db.host=$MONOG_HOST/" /root/apache-tomcat-7.0.75/webapps/drip-api/WEB-INF/classes/drip.properties && sleep 10 && /root/apache-tomcat-7.0.75/bin/shutdown.sh && /root/apache-tomcat-7.0.75/bin/startup.sh && tail -f ./apache-tomcat-7.0.75/logs/catalina.out
+
+
+ENTRYPOINT sed -ie "s/^message.broker.host=.*/message.broker.host=$RABBITMQ_HOST/" /root/apache-tomcat-7.0.75/webapps/drip-api/WEB-INF/classes/drip.properties && sed -ie "s/^db.host=.*/db.host=$MONOG_HOST/" /root/apache-tomcat-7.0.75/webapps/drip-api/WEB-INF/classes/drip.properties && /root/apache-tomcat-7.0.75/bin/shutdown.sh && sleep 20 && mongo -eval 'db.user.insert({"password":"$2a$10$QdysFgsH0sl6Y4BD84UhGO7yyNfoDPXjjEHkDJ3pX6cRfHDj2Q0BO","roles":["ADMIN"],"username":"admin","accountNonExpired":true,"accountNonLocked":true,"credentialsNonExpired":true,"enabled":true})' $MONOG_HOST/drip && /root/apache-tomcat-7.0.75/bin/startup.sh && tail -f /root/apache-tomcat-7.0.75/logs/catalina.out
 
-EXPOSE 8080
-RUN chmod +x startAll.sh
-ENTRYPOINT ./startAll.sh
 
 # To build: docker build -t drip-manager .
 # To run: docker run --name drip-manager-inst -p 127.0.0.1:8080:8080 -d drip-manager

Dockerfiles/planner/Dockerfile

@@ -3,6 +3,8 @@ FROM ubuntu:latest
 
 MAINTAINER S. Koulouzis "S.Koulouzis@uva.nl"
 
+ARG RELESE_VERSION=0.1
+
 
 ENV DEBIAN_FRONTEND=noninteractive 
 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends 
@@ -22,12 +24,12 @@ RUN pip install paramiko
 
 
 
-# Install deployer
+# Install planner
 WORKDIR /root/
-RUN wget https://github.com/QCAPI-DRIP/DRIP-integradation/releases/download/$RELESE_VERSION/drip-deployer.tar.gz
-RUN tar -xzvf drip-deployer.tar.gz
+RUN wget https://github.com/QCAPI-DRIP/DRIP-integradation/releases/download/$RELESE_VERSION/drip-planner.tar.gz
+RUN tar -xzvf drip-planner.tar.gz
 
-ENTRYPOINT python /root/drip-planner/rpc_server.py $RABBITMQ_HOST
+ENTRYPOINT sleep 10 && python /root/drip-planner/rpc_server.py $RABBITMQ_HOST
 
 # docker build -t drip-planner .
 # docker run --name drip-planner-inst -d drip-planner

Dockerfiles/provisioner/Dockerfile

@@ -4,8 +4,8 @@ FROM ubuntu:latest
 MAINTAINER Huan "h.zhou@uva.nl"
 
 
-ARG RABBITMQ_HOST=127.0.0.1
-ARG MONOG_HOST=127.0.0.1
+# ARG RABBITMQ_HOST=127.0.0.1
+# ARG MONOG_HOST=127.0.0.1
 ARG RELESE_VERSION=0.1
 ARG CERTS=/etc/grid-security/certificates
 
@@ -37,12 +37,11 @@ RUN ./fedcloud-ui.sh
 RUN wget https://github.com/QCAPI-DRIP/DRIP-integradation/releases/download/$RELESE_VERSION/drip-provisioner-1.0-SNAPSHOT-jar-with-dependencies.jar
 RUN wget https://github.com/QCAPI-DRIP/DRIP-integradation/releases/download/$RELESE_VERSION/database.tar.gz
 RUN tar -xzvf database.tar.gz
-RUN echo rabbitmq.host=$RABBITMQ_HOST >> provisioner.properties
 RUN echo trusted.certificates.folder=$CERTS >> provisioner.properties
 RUN echo domain.info.path=/root/drip-provisioner >> provisioner.properties
 RUN cat provisioner.properties
 
-ENTRYPOINT java -jar /root/drip-provisioner/drip-provisioner-1.0-SNAPSHOT-jar-with-dependencies.jar /root/drip-provisioner/provisioner.properties
+ENTRYPOINT echo rabbitmq.host=$RABBITMQ_HOST >> /root/drip-provisioner/provisioner.propertie && sleep 10 & java -jar /root/drip-provisioner/drip-provisioner-1.0-SNAPSHOT-jar-with-dependencies.jar /root/drip-provisioner/provisioner.propertie
 
 #Build: docker build -t drip-pro .
 # Run: docker run --name drip-pro-inst -d drip-pro

docs/playbooks/euroArgo_conf_playbook_partition.yml

@@ -3,24 +3,32 @@
   tasks:
   - name: install pre-req
     apt: name="{{ item }}" update_cache=yes state=latest
-    with_items: [python, tcsh, expect, jq , bc,  python-pip, python-dev, build-essential, libgeos-dev, nmap]
+    with_items: [python, tcsh, expect, jq , bc,  python-pip, python-dev, build-essential, libgeos-dev, nmap, git, screen, parallel]
     become: true
-  - {name: pip, pip: name=pyproj}
-  - {name: pip, pip: name=shapely}  
-  - {name: pip, pip: name=psutil}
+  - {name: pip, pip: name=pyproj,become: yes}
+  - {name: pip, pip: name=shapely, become: yes}  
+  - {name: pip, pip: name=psutil, become: yes}
+  - {name: pip pika, pip: name=pika, become: yes}
+  - {name: pip pika, pip: name=klepto, become: yes}
   - {name: mkdir, command: mkfs.ext4 /dev/vdc, become: yes}
-  - {name: create mount point, file: path=/mnt/data/ state=directory owner=vm_user group=vm_user}
-  - {name: create mount point, file: path=/mnt/data/source/ state=directory owner=vm_user group=vm_user}
-  - {name: create mount point, file: path=/mnt/data/source/output state=directory owner=vm_user group=vm_user}
+  - {name: create mount point data, file: path=/mnt/data/ state=directory owner=vm_user group=vm_user}
+  - {name: create mount point source, file: path=/mnt/data/source/ state=directory owner=vm_user group=vm_user}
+  - {name: create mount point output, file: path=/mnt/data/source/output state=directory owner=vm_user group=vm_user}
   - {name: mount, command: mount /dev/vdc /mnt/data, become: yes}
   - {name: download dataset, get_url: 'url=ftp://ftp.ifremer.fr/ifremer/coriolis/co0547-bigdata-archive/envriplus_euroargo_bigdata.tar.gz dest=/mnt/data'}
-  - {name: untar, unarchive: src=/mnt/data/envriplus_euroargo_bigdata.tar.gz dest=/mnt/data/}
-  - {name: download partitioning script, get_url: 'url=https://raw.githubusercontent.com/skoulouzis/dockerfiles/master/ArgoDiffusion/scripts/partitioning.py dest=/mnt/data/source/'}
-  - {name: download getDelta for time_cover, get_url: 'url=https://raw.githubusercontent.com/skoulouzis/dockerfiles/master/ArgoDiffusion/scripts/getDelta.py dest=/mnt/data/source/'}
-  - {name: download ar_bigmetadata for the filter script, get_url: 'url=https://raw.githubusercontent.com/skoulouzis/dockerfiles/master/ArgoDiffusion/scripts/ar_bigmetadata.json dest=/mnt/data/source/'}
-  - {name: download for filter script, get_url: 'url=https://raw.githubusercontent.com/skoulouzis/dockerfiles/master/ArgoDiffusion/scripts/configuration.json dest=/mnt/data/source/'}
-  - {name: exec partitioning, command: python /mnt/data/source/partitioning.py /mnt/data/source/configuration.json, become_user: vm_user}
-  - {name: create data_argo.txt, file: path=//mnt/data/data_argo.txt state=touch owner=vm_user group=vm_user}
+  - {name: untar, unarchive: src=/mnt/data/envriplus_euroargo_bigdata.tar.gz dest=/mnt/data/ remote_src=true}
+  - {name: create workspace, file: path=/home/vm_user/workspace state=directory owner=vm_user group=vm_user}  
+  - {name: clone repo, git: 'repo=https://github.com/skoulouzis/dockerfiles.git dest=/home/vm_user/workspace/dockerfiles version=HEAD force=yes'}
+  - {name: conf script, command: cp /home/vm_user/workspace/dockerfiles/ArgoDiffusion/scripts/ar_bigmetadata.json /mnt/data/source/ar_bigmetadata.json, become: no} 
+  - {name: conf ssh cp, command: cp /etc/ssh/sshd_config /tmp, become: yes}  
+  - {name: conf ssh chown vm_user, command: chown vm_user /tmp/sshd_config, become: yes}
+  - {name: conf ssh echo, command: echo "" >> sshd_config && echo "UseDNS no" >> sshd_config, become: no}
+  - {name: conf ssh chown root, command: chown root /tmp/sshd_config, become: yes}
+  - {name: conf ssh mv, command: mv /tmp/sshd_config /etc/ssh/, become: yes}    
+  - {name: conf ssh restart, command: service ssh restart, become: yes}
+  - {name: screen conf, command: echo "logfile /mnt/data/argoBenchmark" > /home/vm_user/.screenrc, become_user: vm_user}
+  
+
   
   
   
\ No newline at end of file

drip-api/src/main/java/nl/uva/sne/drip/api/service/DeployService.java

@@ -26,13 +26,11 @@ import java.util.Map;
 import java.util.concurrent.TimeoutException;
 import java.util.logging.Level;
 import java.util.logging.Logger;
-import java.util.regex.Matcher;
 import nl.uva.sne.drip.api.dao.DeployDao;
 import nl.uva.sne.drip.api.exception.NotFoundException;
 import nl.uva.sne.drip.api.rpc.DRIPCaller;
 import nl.uva.sne.drip.api.rpc.DeployerCaller;
 import nl.uva.sne.drip.api.v1.rest.DeployController;
-import nl.uva.sne.drip.drip.commons.data.v1.external.CloudCredentials;
 import nl.uva.sne.drip.drip.commons.data.v1.external.DeployRequest;
 import nl.uva.sne.drip.drip.commons.data.v1.external.DeployParameter;
 import nl.uva.sne.drip.drip.commons.data.v1.external.DeployResponse;
@@ -271,7 +269,7 @@ public class DeployService {
                 Map<String, String> nodeTypeCache = new HashMap<>();
                 Map<String, String> domainCache = new HashMap<>();
                 Map<String, String> osTypeCache = new HashMap<>();
-//                Map<String, String> cloudProviderCache = new HashMap<>();
+                Map<String, String> cloudProviderCache = new HashMap<>();
 
                 for (AnsibleOutput ansOut : outputList) {
                     Map<String, Object> map = provisionService.findOne(deployInfo.getProvisionID()).getKeyValue();
@@ -316,6 +314,7 @@ public class DeployService {
                                     }
                                 }
                                 os = (String) component.get("OStype");
+
                                 nodeTypeCache.put(ansOut.getHost(), nodeType);
                                 domainCache.put(ansOut.getHost(), domain);
                                 osTypeCache.put(ansOut.getHost(), os);
@@ -326,7 +325,8 @@ public class DeployService {
                     ansOut.setVmType(nodeType);
                     ansOut.setCloudDeploymentDomain(domain);
                     ansOut.setProvisionID(deployInfo.getProvisionID());
-                    ansOut.setCloudProviderName(os);
+//                    ansOut.setCloudProvider(provider); 
+
                     ansOut = ansibleOutputService.save(ansOut);
                     BenchmarkResult benchmarkResult = parseSaveBenchmarkResult(ansOut);
 

drip-commons/src/main/java/nl/uva/sne/drip/drip/commons/data/v1/external/ansible/AnsibleOutput.java

@@ -105,11 +105,6 @@ public class AnsibleOutput extends OwnedObject {
     public String getProvisionID() {
         return provisionID;
     }
-
-    public void setCloudProviderName(String cloudProvider) {
-        this.setCloudProvider(cloudProvider);
-    }
-
     /**
      * @return the cloudProvider
      */