#!/bin/bash

# Copyright 2019 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -euo pipefail

TEST_ID=${TEST_ID:-$RANDOM}
CLUSTER_NAME=test-cluster-${TEST_ID}

BASE=$(realpath "${BASH_SOURCE[0]}")
BASE_DIR=$(dirname "${BASE}")
TEST_DIR=${BASE_DIR}/ebs-e2e-test
BIN_DIR=${TEST_DIR}/bin
SSH_KEY_PATH=${TEST_DIR}/id_rsa

REGION=${AWS_REGION-us-west-2}
ZONES=${AWS_AVAILABILITY_ZONES:-us-west-2a,us-west-2b,us-west-2c}
INSTANCE_TYPE=${INSTANCE_TYPE:-c4.large}

K8S_VERSION=${K8S_VERSION:-1.18.10}
KOPS_VERSION=${KOPS_VERSION:-1.18.2}
KOPS_STATE_FILE=${KOPS_STATE_FILE:-s3://k8s-kops-csi-e2e}

KUBECONFIG=${KUBECONFIG:-"${HOME}/.kube/config"}
ARTIFACTS=${ARTIFACTS:-"${TEST_DIR}/artifacts"}
GINKGO_FOCUS=${GINKGO_FOCUS:-"\[ebs-csi-migration\]"}
GINKGO_SKIP=${GINKGO_SKIP:-"\[Disruptive\]"}
GINKGO_NODES=${GINKGO_NODES:-4}
CHECK_MIGRATION=${CHECK_MIGRATION:-"true"}

CLEAN=${CLEAN:-"true"}

loudecho() {
  echo "###"
  echo "## ${1}"
  echo "#"
}

loudecho "Testing in region ${REGION} and zones ${ZONES}"
mkdir -p "${BIN_DIR}"
export PATH=${PATH}:${BIN_DIR}

loudecho "Installing kops ${KOPS_VERSION} to ${BIN_DIR}"
source "${BASE_DIR}"/utils/kops.sh
kops::install "${BIN_DIR}" "${KOPS_VERSION}"
KOPS_BIN=${BIN_DIR}/kops

loudecho "Installing helm to ${BIN_DIR}"
source "${BASE_DIR}"/utils/helm.sh
helm::install "${BIN_DIR}"
HELM_BIN=${BIN_DIR}/helm

loudecho "Installing ginkgo to ${BIN_DIR}"
GINKGO_BIN=${BIN_DIR}/ginkgo
if [[ ! -e ${GINKGO_BIN} ]]; then
  pushd /tmp
  GOPATH=${TEST_DIR} GOBIN=${BIN_DIR} GO111MODULE=on go get github.com/onsi/ginkgo/ginkgo@v1.12.0
  popd
fi

AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text)
IMAGE_NAME=${AWS_ACCOUNT_ID}.dkr.ecr.${REGION}.amazonaws.com/aws-ebs-csi-driver
IMAGE_TAG=${TEST_ID}
set +e
if docker images | grep "${IMAGE_NAME}" | grep "${IMAGE_TAG}"; then
  set -e
  loudecho "Assuming ${IMAGE_NAME}:${IMAGE_TAG} has been built and pushed"
else
  set -e
  loudecho "Building and pushing test driver image to ${IMAGE_NAME}:${IMAGE_TAG}"
  eval "$(aws ecr get-login --region "${REGION}" --no-include-email)"
  docker build -t "${IMAGE_NAME}":"${IMAGE_TAG}" .
  docker push "${IMAGE_NAME}":"${IMAGE_TAG}"
fi

loudecho "Generating SSH key $SSH_KEY_PATH"
if [[ ! -e ${SSH_KEY_PATH} ]]; then
  ssh-keygen -P csi-e2e -f "${SSH_KEY_PATH}"
fi

set +e
if ${KOPS_BIN} get cluster --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}".k8s.local; then
  set -e
  loudecho "Updating cluster $CLUSTER_NAME"
else
  set -e
  loudecho "Creating cluster $CLUSTER_NAME"
  ${KOPS_BIN} create cluster --state "${KOPS_STATE_FILE}" \
    --zones "${ZONES}" \
    --node-count=3 \
    --node-size="${INSTANCE_TYPE}" \
    --kubernetes-version="${K8S_VERSION}" \
    --ssh-public-key="${SSH_KEY_PATH}".pub \
    "${CLUSTER_NAME}".k8s.local
fi

CLUSTER_YAML_PATH=${TEST_DIR}/${CLUSTER_NAME}.yaml
${KOPS_BIN} get cluster --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}".k8s.local -o yaml > "${CLUSTER_YAML_PATH}"
cat "${BASE_DIR}"/feature-gates.yaml >> "${CLUSTER_YAML_PATH}"
cat "${BASE_DIR}"/additional-policies.yaml >> "${CLUSTER_YAML_PATH}"
${KOPS_BIN} replace --state "${KOPS_STATE_FILE}" -f "${CLUSTER_YAML_PATH}"
${KOPS_BIN} update cluster --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}".k8s.local --yes

loudecho "Validating cluster $CLUSTER_NAME"
${KOPS_BIN} validate cluster --state "${KOPS_STATE_FILE}" --wait 10m
VALID=$?
if [[ $VALID -ne 0 ]]; then
  exit 1
fi

loudecho "Deploying driver"
${HELM_BIN} upgrade --install aws-ebs-csi-driver \
  --namespace kube-system \
  --set enableVolumeScheduling=true \
  --set enableVolumeResizing=true \
  --set enableVolumeSnapshot=true \
  --set image.repository="${IMAGE_NAME}" \
  --set image.tag="${IMAGE_TAG}" \
  ./charts/aws-ebs-csi-driver

loudecho "Testing focus ${GINKGO_FOCUS}"
set -x
${GINKGO_BIN} -p -nodes="${GINKGO_NODES}" -v --focus="${GINKGO_FOCUS}" --skip="${GINKGO_SKIP}" ./tests/e2e-migration/... -- -kubeconfig="${KUBECONFIG}" -report-dir="${ARTIFACTS}" -gce-zone="${ZONES%,*}"
TEST_PASSED=$?
set +x

if [[ "${CHECK_MIGRATION}" == true ]]; then
  loudecho "Checking migration"
  # There should have been no calls to the in-tree driver kubernetes.io/aws-ebs but many calls to ebs.csi.aws.com
  # Find the controller-manager log and read its metrics to verify
  NODE=$(kubectl get node -l kubernetes.io/role=master -o json | jq -r ".items[].metadata.name")
  kubectl port-forward kube-controller-manager-"${NODE}" 10252:10252 -n kube-system &

  # Ensure port forwarding succeeded
  while true; do
    set +e
    HEALTHZ=$(curl -s 127.0.0.1:10252/healthz)
    set -e
    if [[ ${HEALTHZ} == "ok" ]]; then
      loudecho "Port forwarding succeeded"
      break
    else
      loudecho "Port forwarding is not yet ready"
    fi
    sleep 1
  done

  set +e
  curl 127.0.0.1:10252/metrics -s | grep -a 'volume_operation_total_seconds_bucket{operation_name="provision",plugin_name="ebs.csi.aws.com"'
  CSI_CALLED=${PIPESTATUS[1]}
  set -e

  set +e
  curl 127.0.0.1:10252/metrics -s | grep -a 'volume_operation_total_seconds_bucket{operation_name="provision",plugin_name="kubernetes.io/aws-ebs"'
  INTREE_CALLED=${PIPESTATUS[1]}
  set -e

  for PROC in $(jobs -p); do
    kill "${PROC}"
  done

  loudecho "CSI_CALLED: ${CSI_CALLED}"
  loudecho "INTREE_CALLED: ${INTREE_CALLED}"

  # TEST_PASSED if tests passed, CSI was called, and In-tree was not called
  if [ "${TEST_PASSED}" == 0 ] && [ "${CSI_CALLED}" == 0 ] && [ "${INTREE_CALLED}" == 1 ]; then
    TEST_PASSED=0
  else
    TEST_PASSED=1
  fi
fi

if [[ "${CLEAN}" == true ]]; then
  loudecho "Cleaning"

  loudecho "Removing driver"
  ${HELM_BIN} del aws-ebs-csi-driver

  loudecho "Deleting cluster ${CLUSTER_NAME}"
  ${KOPS_BIN} delete cluster --name "${CLUSTER_NAME}".k8s.local --state "${KOPS_STATE_FILE}" --yes

  rm -rf "${TEST_DIR}"
else
  loudecho "Not cleaning"
fi

loudecho "TEST_PASSED: ${TEST_PASSED}"
if [[ $TEST_PASSED -ne 0 ]]; then
  loudecho "FAIL!"
  exit 1
else
  loudecho "SUCCESS!"
fi
