Vagrantfile

# -*- mode: ruby -*-
# vi: set ft=ruby :

# Copyright 2015 VMware, Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License.  You may obtain a copy of
# the License at http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed
# under the License is distributed on an "AS IS" BASIS, without warranties or
# conditions of any kind, EITHER EXPRESS OR IMPLIED.  See the License for the
# specific language governing permissions and limitations under the License.

Vagrant.configure("2") do |config|

  # ================ VARIABLES ================

  BUILD_DIR="/var/lib/devbox/build"
  ESXCLOUD_DIR="/var/esxcloud"
  DATASTORE_DIR="/tmp/devbox_datastore"
  API_SHARED_SECRET="#{SecureRandom.uuid}"

  SERVICE_PORT_MAP = {
      :management_api => "-p 9000:9000",
      :chairman       => "-p 13000:13000",
      :root_scheduler => "-p 13010:13010",
      :housekeeper    => "-p 16000:16000 -p 16001:16001",
      :cloud_store    => "-p 19000:19000",
      :deployer       => "-p 18000:18000 -p 18001:18001",

  }

  SERVICE_NAME_MAP = {
      :management_api => "management-api",
      :chairman       => "chairman",
      :root_scheduler => "root-scheduler",
      :housekeeper    => "housekeeper",
      :deployer       => "deployer",
      :cloud_store    => "cloud-store"
  }

  BASE_SERVICE_PORT_MAP = {
      :zookeeper      => "-p 2181:2181 -p 2188:2188 -p 3188:3188",
      :haproxy        => "-p 443:443 -p 8080:8080 -p 8088:8088"
  }

  # ================ VM DEFINITIONS ================

  #
  # photon devbox supports virtualbox only for now
  #
  config.vm.provider "virtualbox"

  #
  # VM definition for photon
  #
  config.vm.define "photon" do |photon|
    photon.vm.guest = :photon
    photon.vm.box = ENV['DEVBOX_NAME'] || "photon-devbox-28"
    photon.vm.box_url = ENV['DEVBOX_URL'] ||
        "https://bintray.com/artifact/download/photon-controller/public/28/resource/photon-devbox.box"

    unless ENV["NO_PORT_FORWARDING"]
      # Load Balancer
      photon.vm.network :forwarded_port, guest: 8080, host: 9180,
          auto_correct: true
      # APIFE
      photon.vm.network :forwarded_port, guest: 9000, host: 9080,
          auto_correct: true
      # Graphite
      photon.vm.network :forwarded_port, guest: 8082, host: 9082,
          auto_correct: true
      # Carbon (graphite db backend)
      photon.vm.network :forwarded_port, guest: 2003, host: 9083,
          auto_correct: true
      photon.vm.network :forwarded_port, guest: 2004, host: 9084,
          auto_correct: true
      # Zookeeper
      photon.vm.network :forwarded_port, guest: 2181, host: 9085,
          auto_correct: true
      # Chairman
      photon.vm.network :forwarded_port, guest: 13000, host: 9086,
          auto_correct: true
      # Deployer
      photon.vm.network :forwarded_port, guest: 18000, host: 9088,
          auto_correct: true
      photon.vm.network :forwarded_port, guest: 18001, host: 9089,
          auto_correct: true
      # Load Balancer
      photon.vm.network :forwarded_port, guest: 443, host: 9089,
          auto_correct: true
      photon.vm.network :forwarded_port, guest: 8080, host: 9090,
          auto_correct: true
    end
  end

  #
  # Cleanup containers from previous run to make provision idempotent
  #
  config.vm.provision :shell, :inline => <<-EOS
    set -ex
    echo "Cleaning up devbox..."
    if [ ! -z "$(docker ps -q)" ]
    then
      echo "Stopping containers..."
      docker stop $(docker ps -q)
    fi
    if [ ! -z "$(docker ps -a -q)" ]
    then
      echo "Removing containers..."
      docker rm -v -f $(docker ps -a -q) &> /dev/null
    fi
    echo "Cleanup done."
  EOS

  # ================ NETWORK CONFIGURATION ================

  #
  # Configure network
  #
  if ENV["PUBLIC_NETWORK_IP"]
    public_network_ip = ENV["PUBLIC_NETWORK_IP"]
    network_ip = public_network_ip
    bridge_network = ENV["BRIDGE_NETWORK"]
    public_network_netmask = ENV["PUBLIC_NETWORK_NETMASK"] || "255.255.255.128"
    public_network_netmask = ENV["PUBLIC_NETWORK_NETMASK"]
    config.vm.network :public_network, bridge: bridge_network, ip: public_network_ip, netmask: public_network_netmask

    public_network_gateway = ENV["PUBLIC_NETWORK_GATEWAY"]
    # Assign correct default gateway to public network
    config.vm.provision :shell, :inline => <<-EOS
      set -ex
      ip route del default
      ip route add default via #{public_network_gateway} dev enp0s8
    EOS
  else
    private_network_ip = ENV["PRIVATE_NETWORK_IP"] || "172.31.253.66"
    network_ip = private_network_ip
    config.vm.network :private_network, ip: private_network_ip
  end

  #
  # Configure hostname
  #
  config.vm.hostname = "devbox"

  # ================ PARAMETERS AND SYNC DIRECTORIES ================

  #
  # Configure memory and nat dns
  #
  config.vm.provider "virtualbox" do |v|
    v.customize ["modifyvm", :id, "--memory", ENV["DEVBOX_PHOTON_MEMORY"] || "3072"]
    v.customize ["modifyvm", :id, "--natdnshostresolver1", "on"]
    v.customize ["modifyvm", :id, "--cpus", ENV["DEVBOX_PHOTON_CPUS"] || "4"]
  end

  #
  # Configure sync-ed directories
  #
  config.vm.synced_folder "..", "/devbox_data"
  if Vagrant::VERSION >= '1.3.0'
    config.vm.synced_folder ".", "/vagrant", :mount_options => ["dmode=777", "fmode=666"]
  else
    config.vm.synced_folder ".", "/vagrant", :extra => "dmode=777,fmode=666"
  end
  config.vm.synced_folder "#{Dir.home}/.gradle/#{ENV["JOB_NAME"] || "devbox-local"}", "/gradle", create: true

  #
  # Provided for convenience only; don't take a strong dependency on it
  #
  config.vm.synced_folder Dir.home, "/home_data"

  # ================ PROXY CONFIGURATION ================

  #
  # Our environments need proxies to be set explicitly, we do it by
  # adding /etc/profile.d/proxy.sh script. Vagrant VMs don't pick up
  # those automatically, so we need to copy it to Vagrant VM.
  #
  if ENV["PROXY_PROFILE"]
    proxy_script = "/etc/profile.d/proxy.sh"
    if File.exists?(proxy_script)
      FileUtils.cp(proxy_script, ".")
      config.vm.provision :shell, :inline => <<-EOS
          set -ex
          cp /vagrant/proxy.sh /etc/profile.d
          echo "\nexport no_proxy=#{network_ip},.sock,\\$no_proxy" >> /etc/profile.d/proxy.sh
      EOS
    else
      abort("PROXY_PROFILE is set but '#{proxy_script}' doesn't exist")
    end
  end

  # ================ CONFIGURATION PARAMETERS ================

  dynamic_params = "{"

  #
  # Service-specific
  #
  dynamic_params += "     \"DEPLOYER_REGISTRATION_ADDRESS\" : \"#{network_ip}\""
  dynamic_params += "\n,  \"CHAIRMAN_REGISTRATION_ADDRESS\" : \"#{network_ip}\""
  dynamic_params += "\n,  \"CLOUD-STORE_REGISTRATION_ADDRESS\" : \"#{network_ip}\""
  dynamic_params += "\n,  \"ROOT-SCHEDULER_REGISTRATION_ADDRESS\" : \"#{network_ip}\""
  dynamic_params += "\n,  \"HOUSEKEEPER_REGISTRATION_ADDRESS\" : \"#{network_ip}\""
  dynamic_params += "\n,  \"MANAGEMENT-API_REGISTRATION_ADDRESS\" : \"#{network_ip}\""
  dynamic_params += "\n,  \"TASK_EXPIRATION_THRESHOLD\" : \"3 minutes\""
  dynamic_params += "\n,  \"TASK_EXPIRATION_SCAN_INTERVAL\" : \"3 minutes\""

  #
  # Auth parameters for management API
  #
  dynamic_params += "\n, \"SHARED_SECRET\" : \"#{API_SHARED_SECRET}\""
  if ENV["ENABLE_AUTH"] && ENV["ENABLE_AUTH"] == "true"
    dynamic_params += "\n,  \"ENABLE_AUTH\" : true"

    if ENV["PHOTON_AUTH_LS_ENDPOINT"]
      dynamic_params += "\n,  \"AUTH_SERVER_ADDRESS\" : \"#{ENV["PHOTON_AUTH_LS_ENDPOINT"]}\""
    else
      abort("auth is enabled, but server address is not set")
    end

    if ENV["PHOTON_AUTH_SERVER_PORT"]
      dynamic_params += "\n,  \"AUTH_SERVER_PORT\" : \"#{ENV["PHOTON_AUTH_SERVER_PORT"]}\""
    else
      abort("auth is enabled, but auth server port is not set")
    end

    if ENV["PHOTON_AUTH_SERVER_TENANT"]
      dynamic_params += "\n,  \"AUTH_SERVER_TENANT\" : \"#{ENV["PHOTON_AUTH_SERVER_TENANT"]}\""
    else
      abort("auth is enabled, but auth tenant is not set")
    end

    if ENV["PHOTON_SWAGGER_LOGIN_URL"]
      dynamic_params += "\n,  \"SWAGGER_LOGIN_URL\" : \"#{ENV["PHOTON_SWAGGER_LOGIN_URL"]}\""
    else
      abort("auth is enabled, but swagger login url is not set")
    end

    if ENV["PHOTON_SWAGGER_LOGOUT_URL"]
      dynamic_params += "\n,  \"SWAGGER_LOGOUT_URL\" : \"#{ENV["PHOTON_SWAGGER_LOGOUT_URL"]}\""
    else
      abort("auth is enabled, but swagger logout url is not set")
    end
  else
    dynamic_params += "\n,  \"ENABLE_AUTH\" : false"
  end

  #
  # Postgresql database settings for management API connectivity
  #
  dynamic_params += "\n,  \"DB_HOST\" : \"#{network_ip}\""

  #
  # Real agent vs. fake agent settings
  #
  if ENV["REAL_AGENT"]
    dynamic_params += "\n,  \"USE_ESX_STORE\" : true"

    if ENV["ESX_IP"] && ENV["ESX_DATASTORE"]
      dynamic_params += "\n,  \"ESX_HOST\" : \"#{ENV["ESX_IP"]}\""
      dynamic_params += "\n,  \"DATASTORE\" : \"#{ENV["ESX_DATASTORE"]}\""
    else
      abort("ESX_IP or ESX_DATASTORE not set")
    end
  else
    dynamic_params += "\n,  \"USE_ESX_STORE\" : false"
    dynamic_params += "\n,  \"DATASTORE\" : \"devbox_datastore\""
  end

  #
  # HAProxy
  #
  dynamic_params += "\n,  \"LOAD_BALANCER_SERVERS\": [ { \"serverName\": \"devbox-management-api\", \"serverAddress\": \"#{network_ip}:9000\" } ]"

  #
  # Common
  #
  dynamic_params += "\n,  \"APIFE_IP\" : \"#{network_ip}\""
  dynamic_params += "\n,  \"ZOOKEEPER_QUORUM\" : \"#{network_ip}:2181\""

  if ENV["ENABLE_SYSLOG"] && ENV["ENABLE_SYSLOG"] == "true"
    if ENV["SYSLOG_ENDPOINT"]
      dynamic_params += "\n,  \"SYSLOG_ENDPOINT\" : \"#{ENV["SYSLOG_ENDPOINT"]}\""
      dynamic_params += "\n,  \"ENABLE_SYSLOG\" : true"
    else
      abort("ENABLE_SYSLOG is set, but SYSLOG_ENDPOINT is not")
    end
  else
    dynamic_params += "\n,  \"ENABLE_SYSLOG\" : false"
  end

  dynamic_params += "\n}"

  # ================ DOCKER PREPARATION ================

  config.vm.provision :shell, :inline => <<-EOS
    set -ex

    #
    # Prepare build directory
    #
    if [ ! -d #{BUILD_DIR} ]
    then
      mkdir -p #{BUILD_DIR}
    fi
    cp -r /vagrant/container-artifacts/* #{BUILD_DIR}
    if [ ! -d #{DATASTORE_DIR} ]
    then
      mkdir -p #{DATASTORE_DIR}
    else
      rm -rf #{DATASTORE_DIR}/*
    fi

    #
    # Docker provisioner cannot detect docker is installed if link does not exist.
    #
    if [ ! -f /usr/bin/docker ]
    then
      ln -s /bin/docker /usr/bin/docker
    fi

    #
    # Docker needs a group named 'docker'.
    #
    groupadd -f docker
  EOS

   # ================ VM IMAGE FOR INTEGRATION TEST ================

  #
  # Download management VM image for deployer integration test
  #
  if ENV["DEPLOYER_INTEGRATION_TEST"]
    config.vm.provision :shell, :inline => <<-EOS
      set -ex
      wget -N --no-verbose --no-check-certificate "https://ci.ec.eng.vmware.com/job/machines-stemcell-discus-ova/lastSuccessfulBuild/artifact/management-vm-discus/build/esxcloud-management-vm.ova"
      mkdir -p #{MGMT_IMAGE_DIR}
      tar xvf esxcloud-management-vm.ova esxcloud-management-vm-disk1.vmdk
      cp esxcloud-management-vm-disk1.vmdk #{MGMT_IMAGE_DIR}
      rm -f esxcloud-management-vm-disk1.vmdk
    EOS
  end

  # ================ PROVISIONING ================

  #
  # Give access to zookeeper user inside container to write to log directory in host
  #
  config.vm.provision :shell, :inline => <<-EOS
    set -ex
    chmod -R 777 /var/log/
  EOS

  #
  # Copy repo to local filesystem for mounting into build container
  #
  config.vm.provision :shell, :inline => <<-EOS
    set -ex
    mkdir /esxcloud
    cd /devbox_data
    time tar cf - --exclude=tmp/ --exclude=java/*/build --exclude=.idea --exclude=*.iml --exclude=.ruby . | (cd /esxcloud && tar xf - )
  EOS

  # ================ FAKE AGENT SETUP ================

  datastore_mapped_dir_setting = ""
  datastore_mapped_dir_setting = "-v #{DATASTORE_DIR}:/tmp"
  agent_config_dir=File.join(BUILD_DIR, "agent")

  config.vm.provision :shell, :inline => <<-EOS
    set -ex

    #
    # Build Agent vib
    #
    chmod a+x #{File.join(agent_config_dir, "agent_compile_install.sh")}
    docker run -v /esxcloud:/esxcloud --name agent_compile_install_container \
      -v #{agent_config_dir}:/etc/esxcloud/agent \
      devbox/compileservices /etc/esxcloud/agent/agent_compile_install.sh
    docker wait agent_compile_install_container
    docker commit agent_compile_install_container devbox/agent

    #
    # Copy for use by integration test
    #
    if [ -d /var/esxcloud/packages ]
    then
      rm -f /var/esxcloud/packages/*.vib
    else
      mkdir -p /var/esxcloud/packages
    fi
    cp /esxcloud/python/dist/*.vib /var/esxcloud/packages/

  EOS

  if !(ENV["PUBLIC_NETWORK_IP"] || ENV["REAL_AGENT"])
    agent_parameters = "{\n"
    agent_parameters += "  \"AGENT_BIND_ADDRESS\": \"#{network_ip}\",\n"
    agent_parameters += "  \"AGENT_LOG_PATH\": \"/vagrant/log\",\n"
    agent_parameters += "  \"AGENT_ZOOKEEPER\": \"#{network_ip}:2181\",\n"
    agent_parameters += "  \"AGENT_CHAIRMAN\": \"#{network_ip}:13000\",\n"
    if ENV["ESX_AGENT_COUNT"]
      agent_parameters += ",  \"AGENT_COUNT\": \"#{ENV["ESX_AGENT_COUNT"]}\"\n"
    end
    agent_parameters += "}"

    config.vm.provision :shell, :inline => <<-EOS
      set -ex

      #
      # Create configuration with mustache
      #
      echo 'Agent Parameters file contents:\n#{agent_parameters}'
      echo '#{agent_parameters}' > "#{File.join(agent_config_dir, "agent_parameters.json")}"
      mustache #{File.join(agent_config_dir, "agent_parameters.json")} \
        #{File.join(agent_config_dir, "config.json.template")} > #{File.join(agent_config_dir, "config.json")}
      mustache #{File.join(agent_config_dir, "agent_parameters.json")} \
        #{File.join(agent_config_dir, "run.sh.template")} > #{File.join(agent_config_dir, "run.sh")}
      chmod a+x #{File.join(agent_config_dir, "run.sh")}
      mkdir -p /vagrant/log/agent

      #
      # Start fake agent
      #
      docker run --restart=always -d -p 8835:8835 --net=host --name agent_container \
        -v /vagrant/log/agent:/vagrant/log -v /devbox_data/tmp:/devbox_data/tmp \
        -v #{ESXCLOUD_DIR}:#{ESXCLOUD_DIR} #{datastore_mapped_dir_setting} \
        -v #{agent_config_dir}:/etc/esxcloud/agent \
        devbox/agent /etc/esxcloud/agent/run.sh
    EOS
  end

  #
  # Compile Java
  #
  config.vm.provision :shell, :inline => <<-EOS
    set -ex
    chmod a+x #{File.join(BUILD_DIR, "compile", "compile_java.sh")}
    docker run -v /esxcloud:/esxcloud -v #{BUILD_DIR}:#{BUILD_DIR} -v /gradle:/root/.gradle \
     --name compile_container devbox/compileservices /bin/bash -xe #{File.join(BUILD_DIR, "compile", "compile_java.sh")}
    docker wait compile_container
    docker rm compile_container
  EOS

  # ================ BASE SERVICES ================

  #
  # Create log directories for mounting
  #
  config.vm.provision :shell, :inline => <<-EOS
      set -ex
      if [ ! -d "/vagrant/log/zookeeper" ]
      then
        mkdir -p /vagrant/log/zookeeper
      fi
      if [ ! -d "/vagrant/log/haproxy" ]
      then
        mkdir -p /vagrant/log/haproxy
      fi
  EOS

  #
  # Clean log directories if selected
  #
  if !(ENV["DEVBOX_NO_CLEAN_LOGS"])
    config.vm.provision :shell, :inline => <<-EOS
      set -ex
      rm -rf /vagrant/log/zookeeper/*
      rm -rf /vagrant/log/haproxy/*
    EOS
  end

  #
  # Prepare services configurations
  #
  BASE_SERVICE_PORT_MAP.each do | service, service_ports|
    config_dir="/esxcloud/java/distributions/configurations/configuration-#{service}"
    config.vm.provision :shell, :inline => <<-EOS
      set -ex

      #
      # Process config with mustache
      #
      cd #{config_dir}
      echo '#{dynamic_params}' > dynamic_params.json
      /usr/bin/jq-linux64 -s '.[0] * .[1]' dynamic_params.json #{service}_test.json > #{service}_params.json
      for config_file in #{config_dir}/*.{yml,config,js,sh,sql,cfg}
      do
        if [ -f $config_file ]
        then
          mustache #{service}_params.json $config_file > ${config_file}_temp
          mv ${config_file}_temp $config_file
        fi
      done
    EOS
  end

  #
  # Zookeeper
  #
  config.vm.provision :shell, :inline => <<-EOS
    set -ex
    docker run -d #{BASE_SERVICE_PORT_MAP[:zookeeper]} --restart=always \
      --net=host -v /vagrant/log/zookeeper:/var/log/zookeeper \
      -v /esxcloud/java/distributions/configurations/configuration-zookeeper:/var/esxcloud/data/zookeeper \
      -v /esxcloud/java/distributions/configurations/configuration-zookeeper:/usr/lib/zookeeper/conf \
      --name devbox_zookeeper_container --entrypoint /bin/bash devbox/zookeeper /usr/lib/zookeeper/conf/run.sh
  EOS

  #
  # HAProxy.
  #
  config.vm.provision :shell, :inline => <<-EOS
    set -ex
    docker run -d #{BASE_SERVICE_PORT_MAP[:haproxy]} --restart=always --net=host \
    -v /esxcloud/java/distributions/configurations/configuration-haproxy:/etc/haproxy \
    -v /vagrant/log/haproxy:/var/log --privileged --name devbox_haproxy_container \
    -e HAPROXY_IP="#{network_ip}" --entrypoint /bin/bash devbox/haproxy /etc/haproxy/run.sh
  EOS

  # ================ ESXCLOUD SERVICES ================

  #
  # Build container for each service and start it
  #
  SERVICE_NAME_MAP.each do | service, service_name |
    container_name="devbox_#{service}_container"
    temp_container_name="devbox_#{service}_container_temp"
    image_name="devbox/#{service}"
    config_dir="/esxcloud/java/distributions/configurations/configuration-#{service_name}"

    #
    # Create log directory for mounting
    #
    config.vm.provision :shell, :inline => <<-EOS
      set -ex
      if [ ! -d "/vagrant/log/#{service}" ]
      then
        mkdir -p /vagrant/log/#{service}
      fi
    EOS

    #
    # Clean log directory if selected
    #
    if !(ENV["DEVBOX_NO_CLEAN_LOGS"])
      config.vm.provision :shell, :inline => <<-EOS
        set -ex
        if [ -d "/vagrant/log/#{service}" ]
        then
          rm -rf /vagrant/log/#{service}/*
          mkdir -p /vagrant/log/#{service}
        fi
      EOS
    end

    config.vm.provision :shell, :inline => <<-EOS
      set -ex

      docker run -v /esxcloud/java/distributions:/archive \
        -v #{BUILD_DIR}:#{BUILD_DIR} \
        --name #{temp_container_name} --entrypoint /bin/bash devbox/servicebase #{BUILD_DIR}/service/copy_tars.sh \
          #{service_name}
      docker wait #{temp_container_name}
      docker commit #{temp_container_name} #{image_name}

      #
      # Process config with mustache
      #
      cd #{config_dir}
      echo '#{dynamic_params}' > dynamic_params.json
      /usr/bin/jq-linux64 -s '.[0] * .[1]' dynamic_params.json #{service_name}_test.json > #{service_name}_params.json
      for config_file in #{config_dir}/*.{yml,config,js,sh,sql}
      do
        if [ -f $config_file ]
        then
          mustache #{service_name}_params.json $config_file > ${config_file}_temp
          mv ${config_file}_temp $config_file
        fi
      done

      #
      # Start the container with the service
      #
      docker run --restart=always -d #{SERVICE_PORT_MAP[service]} --net=host --name #{container_name} \
        -v /vagrant/log/#{service}:/vagrant/log -v /devbox_data/tmp:/devbox_data/tmp \
        -v /vagrant/log/#{service}/script_logs:/vagrant/log/script_logs \
        -v #{ESXCLOUD_DIR}:#{ESXCLOUD_DIR} #{datastore_mapped_dir_setting} \
        -v /esxcloud/java/distributions/configurations/configuration-#{service_name}:/etc/esxcloud \
        --entrypoint /bin/bash #{image_name} /etc/esxcloud/run.sh
    EOS
  end

  config.vm.provision :shell, :inline => <<-EOS
    set +e
    echo "Containers:"
    docker ps
    MAX_RETRIES=300
    SLEEP_BETWEEN_RETRIES_SEC=1
    success=-1
    echo "Checking status of services"
    i="0"
    while [ $i -lt $MAX_RETRIES ]
    do
      status=$(curl -s --connect-timeout 60 --max-time 60 --header "Authorization: Bearer #{API_SHARED_SECRET}" http://127.0.0.1:9000/status/)
      if [ '"READY"' == "$(echo $status | jq-linux64 '.status')" ]
      then
        echo "Services started successfully"
        success=0
        break
      fi
      echo "Services not started yet. Status: $status. Retry count: $i"
      sleep $SLEEP_BETWEEN_RETRIES_SEC
      i=$[$i+1]
    done
    exit $success
  EOS

end