#!groovy

pipeline {
    agent { label "minikube-rhel7-8gb" }

    options {
        timestamps()
        timeout(time: 90, unit: 'MINUTES')
        buildDiscarder(logRotator(artifactDaysToKeepStr: '',
                artifactNumToKeepStr: '', daysToKeepStr: '60', numToKeepStr: '100'))
    }

    environment {
        PATH = "/tmp:/qa/tools/opt/apache-maven-3.6.1/bin:/qa/tools/opt/openjdk-11.0.6.10/bin:$PATH"
        JAVA_HOME="/qa/tools/opt/openjdk-11.0.6.10"
        JENKINS_BUILD = "true"

        DEVFILE_PATH = "${WORKSPACE}/test-workspace-devfile.yaml"
        CUSTOM_RESOURCE_PATCH_FILE = "${WORKSPACE}/custom-resource-patch.yaml"

        SUCCESS_THRESHOLD = 5

        LOGS_AND_CONFIGS ="${WORKSPACE}/logs-and-configs"

        mutableCheImageTag = ""
        pathToChectl = ""
    }

    parameters {
        booleanParam(name: 'cheSingleHost',
                defaultValue: false,
                description: 'Configuration Che server strategy')

        string(name: 'cheImageRepo',
                defaultValue: "quay.io/crw_pr/che-server",
                description: 'Repo of Che server image')

        string(name: 'cheImageTag',
                defaultValue: "",
                description: 'Tag of Che server image')

        booleanParam(name: 'buildChe',
                defaultValue: false,
                description: 'Should build Che server')

        string(name: 'e2eTestToRun',
                defaultValue: "test-happy-path",
                description: 'TypeScript E2E test to run')

        string(name: 'testWorkspaceDevfileUrl',
                defaultValue: "",
                description: 'URL of devfile of test workspace')

        booleanParam(name: 'createTestWorkspace',
                defaultValue: true,
                description: 'Should create test workspace')

        string(name: 'ghprbSourceBranch',
                defaultValue: "master")

        string(name: 'e2eTestParameters',
                defaultValue: "")

        string(
                name: "chectlPackageUrl",
                defaultValue: "",
                description: "URL address of chectl package"
        )

        string(
                name: "cheE2eImageTag",
                defaultValue: "nightly",
                description: "Tag of quay.io/eclipse/che-e2e image"
        )
    }

    stages {
        stage("Setup environment") {
            steps {
                script {
                    if (cheImageTag == null || cheImageTag == '') {
                        mutableCheImageTag = ghprbPullId
                    } else {
                        mutableCheImageTag = cheImageTag
                    }
                }
            }
        }

        stage("Prepare to start Che on K8S") {
            failFast true

            parallel {
                stage("Download chectl") {
                    steps {
                        script {
                            retry(2) {
                                if (chectlPackageUrl != "") {
                                    sh """
                                        wget ${chectlPackageUrl} -O chectl-linux-x64.tar.gz
                                        tar -xzf chectl-linux-x64.tar.gz
                                    """
                                    pathToChectl = "${WORKSPACE}/chectl/bin/chectl"

                                } else {
                                    echo "Install chectl using https://www.eclipse.org/che/chectl/ --channel=next"
                                    sh """
                                        curl -sL https://www.eclipse.org/che/chectl/ > install_chectl.sh
                                        chmod +x install_chectl.sh
                                        sudo PATH=$PATH ./install_chectl.sh --channel=next
                                        sudo mv /usr/local/bin/chectl ${WORKSPACE}/chectl
                                        sudo chmod +x ${WORKSPACE}/chectl
                                    """
                                    pathToChectl = "${WORKSPACE}/chectl"
                                }
                            }
                        }
                    }
                }

                stage("Start Kubernetes") {
                    steps {
                        script {
                            retry(2) {
                                echo "Workaround k8s error https://github.com/eclipse/che/issues/14902"
                                sh """
                                    cat /etc/sysctl.conf
                                    echo "net.bridge.bridge-nf-call-iptables = 1" | sudo tee -a /etc/sysctl.conf
                                    sudo sysctl -p
                                """

                                sh """
                                    # https://github.com/kubernetes/minikube/blob/master/docs/vmdriver-none.md
                                    export MINIKUBE_WANTUPDATENOTIFICATION=false
                                    export MINIKUBE_WANTREPORTERRORPROMPT=false
                                    export MINIKUBE_HOME=\$HOME
                                    export CHANGE_MINIKUBE_NONE_USER=true
                                    export KUBECONFIG=\$HOME/.kube/config
                                    export DOCKER_CONFIG=\$HOME/.docker

                                    mkdir -p \$HOME/.kube \$HOME/.minikube
                                    touch \$KUBECONFIG

                                    sudo -E /usr/local/bin/minikube start \\
                                    --vm-driver=none \\
                                    --cpus 2 \\
                                    --memory 6500 \\
                                    --logtostderr
                                """
                            }
                        }
                    }
                }

                stage("Build Che including upstream projects") {
                    when {
                        expression {
                            return buildChe.toString() == 'true' || (params['ghprbCommentBody'] && (params.ghprbCommentBody == 'null' || params.ghprbCommentBody.contains("--rebuild")))
                        }
                    }

                    steps {
                        echo "Build branch ${ghprbSourceBranch} of upstream projects"
                        sh """
                            project_list=(
                                eclipse/che-parent
                            )
                            
                            for project in \${project_list[@]};
                            do
                                result=\$(git ls-remote --heads https://github.com/\${project}.git ${ghprbSourceBranch})
                                if [[ \$result == *"refs"* ]]; then
                                    echo "[PRE-BUILD CHECK] found upstream project: \${project} with same branch name: ${ghprbSourceBranch}"
                                    git clone -b ${ghprbSourceBranch} https://github.com/\${project} \${project}
                                    mvn clean install -f ${WORKSPACE}/\${project}/pom.xml -DskipTests
                                else
                                    echo "[PRE-BUILD CHECK] skip build upstream project: \$project, unable to find branch: ${ghprbSourceBranch}."
                                fi
                            done
                        """

                        echo "Build Che"
                        sh "mvn clean install -f ${WORKSPACE}/pom.xml -T 4 -U -DskipTests -Dskip-enforce -Dskip-validate-sources"
                    }
                }

                stage("Pull test images") {
                    steps {
                        script {
                            retry(2) {
                                echo "Pull test images"
                                sh """
                                    docker pull quay.io/eclipse/che-e2e:$cheE2eImageTag
                                    docker pull quay.io/eclipse/happy-path:nightly
                                """
                            }
                        }
                    }
                }
            }
        }

        stage("Build and push che-server image") {
            when {
                expression {
                    return buildChe.toString() == 'true' || (params['ghprbCommentBody'] && (params.ghprbCommentBody == 'null' || params.ghprbCommentBody.contains("--rebuild")))
                }
            }

            steps {
                withCredentials([string(credentialsId: 'ed71c034-60bc-4fb1-bfdf-9570209076b5', variable: 'maxura_docker_password')]) {
                    sh """
                        echo "Build 'che-server' image"
                        echo "${maxura_docker_password}" | docker login -u maxura --password-stdin

                        ${WORKSPACE}/dockerfiles/che/build.sh --organization:eclipseche --tag:${mutableCheImageTag} --dockerfile:Dockerfile
                    """
                }

                withCredentials([string(credentialsId: 'quay.io-crw_pr+crw_pr_bot_user_token', variable: 'QUAY_TOKEN')]) {
                    sh """
                        echo "Push 'che-server' image"
                        echo "${QUAY_TOKEN}" | docker login -u="crw_pr+crw_pr_bot" --password-stdin quay.io/crw_pr

                        docker tag eclipseche/che-server:${mutableCheImageTag} ${cheImageRepo}:${mutableCheImageTag}
                        echo y | docker push ${cheImageRepo}:${mutableCheImageTag}
                    """
                }
            }
        }

        stage("Start Che") {
            steps {
                script {
                    echo "Pre-pull test images"
                    withCredentials([string(credentialsId: 'ed71c034-60bc-4fb1-bfdf-9570209076b5', variable: 'maxura_docker_password')]) {
                        sh """
                            eval \$(minikube docker-env)

                            docker pull ${cheImageRepo}:${mutableCheImageTag}

                            echo "${maxura_docker_password}" | docker login -u="maxura" --password-stdin docker.io
                            docker pull docker.io/traefik:v2.2.8 || true
                        """
                    }


                    echo "Create custom-resource patch file"
                    if (cheSingleHost.toBoolean() == true) {
                        sh """cat > $CUSTOM_RESOURCE_PATCH_FILE <<EOL
spec:
  server:
    cheImage: '${cheImageRepo}'
    cheImageTag: '${mutableCheImageTag}'
    serverExposureStrategy: 'single-host'
    customCheProperties:
      CHE_LIMITS_USER_WORKSPACES_RUN_COUNT: '-1'
      CHE_WORKSPACE_SIDECAR_IMAGE__PULL__POLICY: IfNotPresent
      CHE_WORKSPACE_PLUGIN__BROKER_PULL__POLICY: IfNotPresent
      CHE_INFRA_KUBERNETES_WORKSPACE_START_TIMEOUT_MIN: '12'
  k8s:  
    singleHostExposureType: 'gateway'
  auth:
    updateAdminPassword: false
EOL"""

                    } else {
                        sh """cat > $CUSTOM_RESOURCE_PATCH_FILE <<EOL
spec:
  server:
    cheImage: '${cheImageRepo}'
    cheImageTag: '${mutableCheImageTag}'
    customCheProperties:
      CHE_LIMITS_USER_WORKSPACES_RUN_COUNT: '-1'
      CHE_WORKSPACE_SIDECAR_IMAGE__PULL__POLICY: IfNotPresent
      CHE_WORKSPACE_PLUGIN__BROKER_PULL__POLICY: IfNotPresent
      CHE_INFRA_KUBERNETES_WORKSPACE_START_TIMEOUT_MIN: '12'
  auth:
    updateAdminPassword: false
EOL"""
                    }

                    echo "Install Che"
                    sh """
                        ${pathToChectl} server:deploy \\
                        --k8spodreadytimeout=600000 \\
                        --k8spodwaittimeout=600000 \\
                        --k8spoddownloadimagetimeout=600000 \\
                        --listr-renderer=verbose \\
                        --platform=minikube \\
                        --che-operator-cr-patch-yaml=$CUSTOM_RESOURCE_PATCH_FILE \\
                        --telemetry=off \\
                        --chenamespace=eclipse-che
                    """

                    cheHost = sh(
                            script: "kubectl get ingress che --namespace=eclipse-che -o=jsonpath={'.spec.rules[0].host'}",
                            returnStdout: true
                    ).trim()

                    echo "Wait che-server to be available"
                    sh """
                            COUNTER=0;
                            SUCCESS_RATE_COUNTER=0;
                            while true; do
                            if [ \$COUNTER -gt 180 ]; then
                            echo "Unable to get stable route. Exiting"
                            exit 1
                            fi
                            
                            ((COUNTER+=1))
                            
                            
                            STATUS_CODE=\$(curl -k -sL -w "%{http_code}" -I http://${cheHost} -o /dev/null; true) || true
                            
                            echo "Try \${COUNTER}. Status code: \${STATUS_CODE}"
                            if [ "\$STATUS_CODE" == "200" ]; then 
                            ((SUCCESS_RATE_COUNTER+=1))
                            fi
                            sleep 1;
                        
                            if [ \$SUCCESS_RATE_COUNTER == \$SUCCESS_THRESHOLD ]; then 
                            echo "Route is stable enough. Continuing running tests"
                            break
                            fi
                            done
                    """
                }
            }
        }

        stage("Create test workspace") {
            when {
                expression { return createTestWorkspace.toString() == 'true' }
            }

            steps {
                script {
                    if (testWorkspaceDevfileUrl != null && testWorkspaceDevfileUrl != "") {
                        sh "wget $testWorkspaceDevfileUrl -O $DEVFILE_PATH"

                    } else {
                        sh "cp ${WORKSPACE}/tests/e2e/files/happy-path/happy-path-workspace.yaml $DEVFILE_PATH"
                    }

                    sh """
                        ${pathToChectl} auth:login -u admin -p admin --telemetry=off --chenamespace=eclipse-che

                        ${pathToChectl} workspace:create --start \\
                           --devfile=${DEVFILE_PATH} \\
                           --telemetry=off \\
                           --chenamespace=eclipse-che
                    """
                }
            }
        }

        stage("Run E2E Happy path tests") {
            steps {
                sh """
                     CHE_URL=https://${cheHost}
                     docker run --shm-size=1g --net=host --ipc=host \\
                       -p 5920:5920 \\
                       -e TS_SELENIUM_BASE_URL=\${CHE_URL} \\
                       -e TS_SELENIUM_MULTIUSER="true" \\
                       -e TS_SELENIUM_USERNAME="admin" \\
                       -e TS_SELENIUM_PASSWORD="admin" \\
                       -e TEST_SUITE="${e2eTestToRun}" ${e2eTestParameters} \\
                       -e NODE_TLS_REJECT_UNAUTHORIZED=0 \\
                       -e TS_SELENIUM_LOG_LEVEL=TRACE \\
                       -e TS_SELENIUM_START_WORKSPACE_TIMEOUT=720000 \\
                       -v ${WORKSPACE}/tests/e2e:/tmp/e2e:Z \\
                       quay.io/eclipse/che-e2e:${cheE2eImageTag}
                """
            }
        }
    }

    post {
        cleanup {
            sh """                   
                mkdir -p $LOGS_AND_CONFIGS/kubectl
                kubectl --namespace=eclipse-che get events > $LOGS_AND_CONFIGS/kubectl/events.txt || true
                kubectl --namespace=eclipse-che get events -o json > $LOGS_AND_CONFIGS/kubectl/events.json || true
                kubectl get pods -A >$LOGS_AND_CONFIGS/kubectl/pods.txt || true

                mkdir -p $LOGS_AND_CONFIGS/che-config

                cp ${WORKSPACE}/org_v1_che_cr.yaml $LOGS_AND_CONFIGS/che-config || true
                cp ${DEVFILE_PATH} $LOGS_AND_CONFIGS || true
                cp ${CUSTOM_RESOURCE_PATCH_FILE} $LOGS_AND_CONFIGS || true

                kubectl get checluster --namespace=eclipse-che -o yaml > $LOGS_AND_CONFIGS/che-config/checluster.yaml || true
                kubectl get configmaps --namespace=eclipse-che che -o yaml > $LOGS_AND_CONFIGS/che-config/configmap.yaml || true
                kubectl get ingresses --namespace=eclipse-che > $LOGS_AND_CONFIGS/che-config/ingresses.txt || true
                kubectl get pods --namespace=eclipse-che >$LOGS_AND_CONFIGS/che-config/pods.txt || true
                kubectl get all --namespace=eclipse-che -o yaml > $LOGS_AND_CONFIGS/che-config/all.yaml || true

                mkdir -p $LOGS_AND_CONFIGS/che-logs
                export PODS=\$(kubectl get pod --namespace=eclipse-che | awk 'NR>1 {print \$1}')
                for pod in \$PODS ; do \
                    kubectl logs --namespace=eclipse-che "\${pod}" --namespace=eclipse-che > $LOGS_AND_CONFIGS/che-logs/"\${pod}".pod.log || true; \
                done
                kubectl describe pod --namespace=eclipse-che > $LOGS_AND_CONFIGS/che-config/eclipse-che-pod-description.txt || true

                mkdir -p $LOGS_AND_CONFIGS/workspace-logs
                WORKSPACE_NAMESPACE_NAME=admin-che
                export WS_POD=\$(kubectl get pod --namespace=\$WORKSPACE_NAMESPACE_NAME | grep ".workspace-" | awk '{print \$1}')
                for c in \$(kubectl get pod --namespace=\$WORKSPACE_NAMESPACE_NAME \$WS_POD -o jsonpath="{.spec.containers[*].name}") ; do
                    CONTAINER_LOGS_DIR=$LOGS_AND_CONFIGS/workspace-logs/"\${c}"
                    mkdir \$CONTAINER_LOGS_DIR
                    kubectl logs \$WS_POD "\${c}" --namespace=\$WORKSPACE_NAMESPACE_NAME > \${CONTAINER_LOGS_DIR}.container.log || true;
                    kubectl cp \$WORKSPACE_NAMESPACE_NAME/\$WS_POD:/workspace_logs \$CONTAINER_LOGS_DIR -c "\${c}" || true;
                done
                kubectl describe pod --namespace=\$WORKSPACE_NAMESPACE_NAME \$WS_POD > $LOGS_AND_CONFIGS/che-config/workspace-pod-description.txt || true

                mkdir -p $LOGS_AND_CONFIGS/docker
                docker image ls > $LOGS_AND_CONFIGS/docker/images.txt || true
                docker ps -a > $LOGS_AND_CONFIGS/docker/containers.txt || true

                env > $LOGS_AND_CONFIGS/env.output.txt || true
                ${pathToChectl} --help > $LOGS_AND_CONFIGS/chectl.version.txt || true

                cp -r /tmp/chectl-logs $LOGS_AND_CONFIGS || true
                cp /home/hudson/.cache/chectl/error.log /tmp/chectl-logs || true
            """

            archiveArtifacts allowEmptyArchive: true, artifacts: "tests/e2e/report/**, logs-and-configs/**"
        }

    }

}