#!/bin/bash

# WARNING: The script modifies the host on which it is run.  It loads
# the openvwitch and br_netfilter modules and sets
# net.bridge.bridge-nf-call-iptables=0.  Consider creating dind
# clusters in a VM if this modification is undesirable:
#
#   OPENSHIFT_DIND_DEV_CLUSTER=1 vagrant up'
#
# Overview
# ========
#
# This script manages the lifecycle of an openshift dev cluster
# deployed to docker-in-docker containers.  Once 'start' has been used
# to successfully create a dind cluster, 'docker exec' can be used to
# access the created containers (named
# openshift-{master,node-1,node-2}) as if they were VMs.
#
# Dependencies
# ------------
#
# This script has been tested on Fedora 21, but should work on any
# release.  Docker is assumed to be installed.  At this time,
# boot2docker is not supported.
#
# SELinux
# -------
#
# Docker-in-docker's use of volumes is not compatible with selinux set
# to enforcing mode.  Set selinux to permissive or disable it
# entirely.
#
# OpenShift Configuration
# -----------------------
#
# By default, a dind openshift cluster stores its configuration
# (openshift.local.*) in /tmp/openshift-dind-cluster/openshift.  Since
# configuration is stored in a different location than a
# vagrant-deployed cluster (which stores configuration in the root of
# the origin tree), vagrant and dind clusters can run simultaneously
# without conflict.  It's also possible to run multiple dind clusters
# simultaneously by overriding the instance prefix.  The following
# command would ensure configuration was stored at
# /tmp/openshift-dind/cluster/my-cluster:
#
#    OPENSHIFT_INSTANCE_PREFIX=my-cluster hack/dind-cluster.sh [command]
#
# It is also possible to specify an entirely different configuration path:
#
#    OPENSHIFT_CONFIG_ROOT=[path] hack/dind-cluster.sh [command]
#
# Suggested Workflow
# ------------------
#
# When making changes to the deployment of a dind cluster or making
# breaking golang changes, the 'restart' command will ensure that an
# existing cluster is cleaned up before deploying a new cluster.
#
# When only making non-breaking changes to golang code, the 'redeploy'
# command avoids restarting the cluster.  'redeploy' rebuilds the
# openshift binaries and deploys them to the existing cluster.
#
# Running Tests
# -------------
#
# The extended tests can be run against a dind cluster as follows:
#
#     OPENSHIFT_CONFIG_ROOT=dind test/extended/networking.sh

set -o errexit
set -o nounset
set -o pipefail

DIND_MANAGEMENT_SCRIPT=true

source $(dirname "${BASH_SOURCE}")/../contrib/vagrant/provision-config.sh

# Enable xtrace for container script invocations if it is enabled
# for this script.
BASH_CMD=
if set +o | grep -q '\-o xtrace'; then
  BASH_CMD="bash -x"
fi

DOCKER_CMD=${DOCKER_CMD:-"sudo docker"}

# Override the default CONFIG_ROOT path with one that is
# cluster-specific.
TMPDIR="${TMPDIR:-"/tmp"}"
CONFIG_ROOT=${OPENSHIFT_CONFIG_ROOT:-${TMPDIR}/openshift-dind-cluster/${INSTANCE_PREFIX}}

DEPLOY_SSH=${OPENSHIFT_DEPLOY_SSH:-true}

DEPLOYED_CONFIG_ROOT="/config"

DEPLOYED_ROOT="/data/src/github.com/openshift/origin"

SCRIPT_ROOT="${DEPLOYED_ROOT}/contrib/vagrant"

function check-selinux() {
  if [[ "$(getenforce)" = "Enforcing" ]]; then
    >&2 echo "Error: This script is not compatible with SELinux enforcing mode."
    exit 1
  fi
}

DIND_IMAGE="openshift/dind"
BUILD_IMAGES="${OPENSHIFT_DIND_BUILD_IMAGES:-1}"

function build-image() {
  local build_root=$1
  local image_name=$2

  pushd "${build_root}" > /dev/null
    ${DOCKER_CMD} build -t "${image_name}" .
  popd > /dev/null
}

function build-images() {
  # Building images is done by default but can be disabled to allow
  # separation of image build from cluster creation.
  if [[ "${BUILD_IMAGES}" = "1" ]]; then
    echo "Building container images"
    build-image "${OS_ROOT}/images/dind" "${DIND_IMAGE}"
  fi
}

function get-docker-ip() {
  local cid=$1

  ${DOCKER_CMD} inspect --format '{{ .NetworkSettings.IPAddress }}' "${cid}"
}

function docker-exec-script() {
    local cid=$1
    local cmd=$2

    ${DOCKER_CMD} exec -t "${cid}" ${BASH_CMD} ${cmd}
}

function start() {
  # docker-in-docker's use of volumes is not compatible with SELinux
  check-selinux

  echo "Configured network plugin: ${NETWORK_PLUGIN}"

  # TODO(marun) - perform these operations in a container for boot2docker compat
  echo "Ensuring compatible host configuration"
  sudo modprobe openvswitch
  sudo modprobe br_netfilter 2> /dev/null || true
  sudo sysctl -w net.bridge.bridge-nf-call-iptables=0 > /dev/null
  # overlayfs, if available, will be faster than vfs
  sudo modprobe overlay 2> /dev/null || true
  mkdir -p "${CONFIG_ROOT}"

  if [[ "${SKIP_BUILD}" = "true" ]]; then
    echo "WARNING: Skipping image build due to OPENSHIFT_SKIP_BUILD=true"
  else
    build-images
  fi

  ## Create containers
  echo "Launching containers"
  local root_volume="-v ${OS_ROOT}:${DEPLOYED_ROOT}"
  local config_volume="-v ${CONFIG_ROOT}:${DEPLOYED_CONFIG_ROOT}"
  local volumes="${root_volume} ${config_volume}"
  # systemd requires RTMIN+3 to shutdown properly
  local stop="--stop-signal=$(kill -l RTMIN+3)"
  local base_run_cmd="${DOCKER_CMD} run -dt ${stop} ${volumes}"

  local master_cid="$(${base_run_cmd} --privileged --name="${MASTER_NAME}" \
      --hostname="${MASTER_NAME}" "${DIND_IMAGE}")"
  local master_ip="$(get-docker-ip "${master_cid}")"

  local node_cids=()
  local node_ips=()
  for name in "${NODE_NAMES[@]}"; do
    local cid="$(${base_run_cmd} --privileged --name="${name}" \
        --hostname="${name}" "${DIND_IMAGE}")"
    node_cids+=( "${cid}" )
    node_ips+=( "$(get-docker-ip "${cid}")" )
  done
  node_ips="$(os::provision::join , ${node_ips[@]})"

  ## Provision containers
  local args="${master_ip} ${NODE_COUNT} ${node_ips} ${INSTANCE_PREFIX} \
-n ${NETWORK_PLUGIN}"
  if [[ "${SKIP_BUILD}" = "true" ]]; then
      args="${args} -s"
  fi

  echo "Provisioning ${MASTER_NAME}"
  local cmd="${SCRIPT_ROOT}/provision-master.sh ${args} -c \
${DEPLOYED_CONFIG_ROOT}"
  docker-exec-script "${master_cid}" "${cmd}"

  if [[ "${DEPLOY_SSH}" = "true" ]]; then
    ${DOCKER_CMD} exec -t "${master_cid}" ssh-keygen -N '' -q -f /root/.ssh/id_rsa
    cmd="cat /root/.ssh/id_rsa.pub"
    local public_key="$(${DOCKER_CMD} exec -t "${master_cid}" ${cmd})"
    cmd="cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys"
    ${DOCKER_CMD} exec -t "${master_cid}" ${cmd}
    ${DOCKER_CMD} exec -t "${master_cid}" systemctl start sshd
  fi

  # Ensure that all users (e.g. outside the container) have read-write
  # access to the openshift configuration.  Security shouldn't be a
  # concern for dind since it should only be used for dev and test.
  local openshift_config_path="${CONFIG_ROOT}/openshift.local.config"
  find "${openshift_config_path}" -exec sudo chmod ga+rw {} \;
  find "${openshift_config_path}" -type d -exec sudo chmod ga+x {} \;

  for (( i=0; i < ${#node_cids[@]}; i++ )); do
    local node_index=$((i + 1))
    local cid="${node_cids[$i]}"
    local name="${NODE_NAMES[$i]}"
    echo "Provisioning ${name}"
    cmd="${SCRIPT_ROOT}/provision-node.sh ${args} -i ${node_index} -c \
${DEPLOYED_CONFIG_ROOT}"
    docker-exec-script "${cid}" "${cmd}"

    if [[ "${DEPLOY_SSH}" = "true" ]]; then
      ${DOCKER_CMD} exec -t "${cid}" mkdir -p /root/.ssh
      cmd="echo ${public_key} > /root/.ssh/authorized_keys"
      ${DOCKER_CMD} exec -t "${cid}" bash -c "${cmd}"
      ${DOCKER_CMD} exec -t "${cid}" systemctl start sshd
    fi
  done

  local rc_file="dind-${INSTANCE_PREFIX}.rc"
  local admin_config="$(os::provision::get-admin-config ${CONFIG_ROOT})"
  local bin_path="$(os::build::get-bin-output-path "${OS_ROOT}")"
  cat >"${rc_file}" <<EOF
export KUBECONFIG=${admin_config}
export PATH=\$PATH:${bin_path}
EOF

  # Disable the sdn node as late as possible to allow time for the
  # node to register itself.
  if [[ "${SDN_NODE}" = "true" ]]; then
    os::provision::disable-node "${OS_ROOT}" "${CONFIG_ROOT}" \
        "${SDN_NODE_NAME}"
  fi

  if [[ "${KUBECONFIG:-}" != "${admin_config}"  ||
          ":${PATH}:" != *":${bin_path}:"* ]]; then
    echo ""
    echo "Before invoking the openshift cli, make sure to source the
cluster's rc file to configure the bash environment:

  $ . ${rc_file}
  $ oc get nodes
"
  fi
}

function stop() {
  echo "Cleaning up docker-in-docker containers"

  local master_cid="$(${DOCKER_CMD} ps -qa --filter "name=${MASTER_NAME}")"
  if [[ "${master_cid}" ]]; then
    ${DOCKER_CMD} rm -f "${master_cid}"
  fi

  local node_cids="$(${DOCKER_CMD} ps -qa --filter "name=${NODE_PREFIX}")"
  if [[ "${node_cids}" ]]; then
    node_cids=(${node_cids//\n/ })
    for cid in "${node_cids[@]}"; do
      ${DOCKER_CMD} rm -f "${cid}"
    done
  fi

  echo "Cleanup up configuration to avoid conflict with a future cluster"
  # The container will have created configuration as root
  sudo rm -rf ${CONFIG_ROOT}/openshift.local.*

  # Cleanup orphaned volumes
  #
  # See: https://github.com/jpetazzo/dind#important-warning-about-disk-usage
  #
  echo "Cleaning up volumes used by docker-in-docker daemons"
  local volume_ids=$(${DOCKER_CMD} volume ls -qf dangling=true)
  if [[ "${volume_ids}" ]]; then
    ${DOCKER_CMD} volume rm ${volume_ids}
  fi
}

# Build and deploy openshift binaries to an existing cluster
function redeploy() {
  local node_service="openshift-node"

  ${DOCKER_CMD} exec -t "${MASTER_NAME}" bash -c "\
. ${SCRIPT_ROOT}/provision-util.sh ; \
os::provision::build-origin ${DEPLOYED_ROOT} ${SKIP_BUILD}"

  echo "Stopping ${MASTER_NAME} service(s)"
  ${DOCKER_CMD} exec -t "${MASTER_NAME}" systemctl stop "${MASTER_NAME}"
  if [[ "${SDN_NODE}" = "true" ]]; then
    ${DOCKER_CMD} exec -t "${MASTER_NAME}" systemctl stop "${node_service}"
  fi
  echo "Updating ${MASTER_NAME} binaries"
  ${DOCKER_CMD} exec -t "${MASTER_NAME}" bash -c \
". ${SCRIPT_ROOT}/provision-util.sh ; \
os::provision::install-cmds ${DEPLOYED_ROOT}"
  echo "Starting ${MASTER_NAME} service(s)"
  ${DOCKER_CMD} exec -t "${MASTER_NAME}" systemctl start "${MASTER_NAME}"
  if [[ "${SDN_NODE}" = "true" ]]; then
    ${DOCKER_CMD} exec -t "${MASTER_NAME}" systemctl start "${node_service}"
  fi

  for node_name in "${NODE_NAMES[@]}"; do
    echo "Stopping ${node_name} service"
    ${DOCKER_CMD} exec -t "${node_name}" systemctl stop "${node_service}"
    echo "Updating ${node_name} binaries"
    ${DOCKER_CMD} exec -t "${node_name}" bash -c "\
. ${SCRIPT_ROOT}/provision-util.sh ; \
os::provision::install-cmds ${DEPLOYED_ROOT}"
    echo "Starting ${node_name} service"
    ${DOCKER_CMD} exec -t "${node_name}" systemctl start "${node_service}"
  done
}

function nodes-are-ready() {
  local oc="$(os::build::find-binary oc)"
  local kc="$(os::provision::get-admin-config ${CONFIG_ROOT})"
  read -d '' template <<'EOF'
{{range $item := .items}}
  {{if not .spec.unschedulable}}
    {{range .status.conditions}}
      {{if eq .type "Ready"}}
        {{if eq .status "True"}}
          {{printf "%s\\n" $item.metadata.name}}
        {{end}}
      {{end}}
    {{end}}
  {{end}}
{{end}}
EOF
  # Remove formatting before use
  template="$(echo "${template}" | tr -d '\n' | sed -e 's/} \+/}/g')"
  local count="$("${oc}" --config="${kc}" get nodes \
                         --template "${template}" | wc -l)"
  test "${count}" -ge "${NODE_COUNT}"
}

function wait-for-cluster() {
  local msg="nodes to register with the master"
  local condition="nodes-are-ready"
  os::provision::wait-for-condition "${msg}" "${condition}"
}

case "${1:-""}" in
  start)
    start
    ;;
  stop)
    stop
    ;;
  restart)
    stop
    start
    ;;
  redeploy)
    redeploy
    ;;
  wait-for-cluster)
    wait-for-cluster
    ;;
  build-images)
    BUILD_IMAGES=1
    build-images
    ;;
  config-host)
    os::provision::set-os-env "${OS_ROOT}" "${CONFIG_ROOT}"
    ;;
  *)
    echo "Usage: $0 {start|stop|restart|redeploy|wait-for-cluster|build-images|config-host}"
    exit 2
esac
