diff --git a/jobs/mini-e2e.yaml b/jobs/mini-e2e.yaml new file mode 100644 index 000000000..0778a14f6 --- /dev/null +++ b/jobs/mini-e2e.yaml @@ -0,0 +1,35 @@ +--- +- job: + name: mini-e2e + project-type: pipeline + sandbox: true + concurrent: true + properties: + - github: + url: https://github.com/ceph/ceph-csi + - build-discarder: + days-to-keep: 7 + artifact-days-to-keep: 7 + pipeline-scm: + scm: + - git: + name: origin + url: https://github.com/ceph/ceph-csi + branches: + - ci/centos + script-path: mini-e2e.groovy + lightweight-checkout: true + triggers: + - github-pull-request: + status-context: ci/centos/mini-e2e + trigger-phrase: '/(re)?test ((all)|(ci/centos/mini-e2e))' + permit-all: true + # TODO: set github-hooks to true when it is configured in GitHub + github-hooks: false + cron: 'H/5 * * * *' + black-list-target-branches: + - ci/centos + admin-list: + - nixpanic + org-list: + - ceph diff --git a/mini-e2e.groovy b/mini-e2e.groovy new file mode 100644 index 000000000..2ab50685f --- /dev/null +++ b/mini-e2e.groovy @@ -0,0 +1,73 @@ +def cico_retries = 16 +def cico_retry_interval = 60 +// temporary git repository for testing purpose +def ci_git_repo = 'https://github.com/nixpanic/ceph-csi' +def ci_git_branch = 'mini-e2e' +def git_repo = 'https://github.com/ceph/ceph-csi' +def ref = "master" + +def ssh(cmd) { + sh "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@${CICO_NODE} '${cmd}'" +} + +node('cico-workspace') { + stage('checkout ci repository') { + git url: "${ci_git_repo}", + branch: "${ci_git_branch}", + changelog: false + } + + stage('reserve bare-metal machine') { + def firstAttempt = true + retry(30) { + if (!firstAttempt) { + sleep(time: 5, unit: "MINUTES") + } + firstAttempt = false + cico = sh( + script: "cico node get -f value -c hostname -c comment --release=8 --retry-count=${cico_retries} --retry-interval=${cico_retry_interval}", + returnStdout: true + ).trim().tokenize(' ') + env.CICO_NODE = "${cico[0]}.ci.centos.org" + env.CICO_SSID = "${cico[1]}" + } + } + + try { + stage('prepare bare-metal machine') { + if (params.ghprbPullId != null) { + ref = "pull/${ghprbPullId}/head" + } + sh 'scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ./prepare.sh ./single-node-k8s.sh root@${CICO_NODE}:' + ssh "./prepare.sh --workdir=/opt/build/go/src/github.com/ceph/ceph-csi --gitrepo=${git_repo} --ref=${ref}" + } + stage('build artifacts') { + // build container image + ssh 'cd /opt/build/go/src/github.com/ceph/ceph-csi && make image-cephcsi GOARCH=amd64 CONTAINER_CMD=podman' + // build e2e.test executable + ssh 'cd /opt/build/go/src/github.com/ceph/ceph-csi && make containerized-build CONTAINER_CMD=podman TARGET=e2e.test' + } + stage('deploy k8s v1.18.3 and rook') { + timeout(time: 30, unit: 'MINUTES') { + ssh './single-node-k8s.sh --k8s-version=v1.18.3' + } + } + stage('run e2e') { + timeout(time: 60, unit: 'MINUTES') { + ssh 'cd /opt/build/go/src/github.com/ceph/ceph-csi && make run-e2e' + } + } + } + + catch(exc) { + stage('debug time!') { + ssh 'sleep 8h' + } + } + + finally { + stage('return bare-metal machine') { + sh 'cico node done ${CICO_SSID}' + } + } +} diff --git a/single-node-k8s.sh b/single-node-k8s.sh new file mode 100755 index 000000000..15fe81723 --- /dev/null +++ b/single-node-k8s.sh @@ -0,0 +1,147 @@ +#!/bin/bash +# +# Start minukube with the selected k8s version and deploy rook. +# +# This script uses helper scripts from the main ceph-csi branch. +# + +# fail when a command returns an error +set -e -o pipefail + +ARGUMENT_LIST=( + "k8s-version" +) + +opts=$(getopt \ + --longoptions "$(printf "%s:," "${ARGUMENT_LIST[@]}")help" \ + --name "$(basename "${0}")" \ + --options "" \ + -- "$@" +) +ret=$? + +if [ ${ret} -ne 0 ] +then + echo "Try '--help' for more information." + exit 1 +fi + +eval set -- "${opts}" + +while true; do + case "${1}" in + --help) + shift + echo "Options:" + echo "--help|-h specify the flags" + echo "--k8s-version specify the kubernetes version" + + echo " " + echo "Sample Usage:" + echo "./single-node-k8s.sh --k8s-version=v1.18.3" + exit 0 + ;; + --k8s-version) + shift + k8s_version=${1} + ;; + --) + shift + break + ;; + esac + shift +done + +test -n "${k8s_version}" || { echo "k8s_version is not set"; exit 1;} + +function set_env() { + export GOPATH="/opt/build/go" + # build.env is not part of the ci/centos branch, so shellcheck can not find + # it and will cause a failure if not annotated with /dev/null. + # shellcheck source=/dev/null + source "${GOPATH}"/src/github.com/ceph/ceph-csi/build.env + export GO111MODULE="on" + export TEST_COVERAGE="stdout" + export VM_DRIVER="kvm2" + export CEPH_CSI_RUN_ALL_TESTS=true + # downloading rook images is sometimes slow, extend timeout to 15 minutes + export ROOK_DEPLOY_TIMEOUT=900 + + # script/minikube.sh installs under /usr/local/bin + export PATH=$PATH:/usr/local/bin +} + +# FIXME: unfortunately minikube does not work with podman (yet) +function install_docker() +{ + curl https://download.docker.com/linux/centos/docker-ce.repo -o /etc/yum.repos.d/docker-ce.repo + dnf -y --nobest install docker-ce + systemctl enable --now docker +} + +function install_minikube() +{ + dnf -y groupinstall 'Virtualization Host' + systemctl enable --now libvirtd + # Warning about "No ACPI IVRS table found", not critical + virt-host-validate || true + + # minikube needs socat + dnf -y install socat + + # deploy minikube + MINIKUBE_VERSION="${MINIKUBE_VERSION}" KUBE_VERSION="${k8s_version}" ${GOPATH}/src/github.com/ceph/ceph-csi/scripts/minikube.sh up + + # copy kubectl from minikube to /usr/bin + cp ~/.minikube/cache/linux/"${k8s_version}"/kubectl /usr/bin/ + + # add disks to the minikube VM + if [ ! -d /opt/minikube/images ]; then + mkdir -p /opt/minikube/images + virsh pool-create-as --name minikube --type dir --target /opt/minikube/images + fi + virsh vol-create-as --pool minikube --name osd-0 --capacity 32G + virsh vol-create-as --pool minikube --name osd-1 --capacity 32G + virsh vol-create-as --pool minikube --name osd-2 --capacity 32G + virsh attach-disk --domain minikube --source /opt/minikube/images/osd-0 --target vdb + virsh attach-disk --domain minikube --source /opt/minikube/images/osd-1 --target vdc + virsh attach-disk --domain minikube --source /opt/minikube/images/osd-2 --target vdd + # rescan for newly attached virtio devices + minikube ssh 'echo 1 | sudo tee /sys/bus/pci/rescan > /dev/null ; dmesg | grep virtio_blk' +} + +function deploy_rook() +{ + ${GOPATH}/src/github.com/ceph/ceph-csi/scripts/minikube.sh deploy-rook + ${GOPATH}/src/github.com/ceph/ceph-csi/scripts/minikube.sh create-block-pool + ${GOPATH}/src/github.com/ceph/ceph-csi/scripts/minikube.sh k8s-sidecar + + # TODO: only needed for k8s 1.17.0 and newer + # delete snapshot CRD created by ceph-csi in rook + ${GOPATH}/src/github.com/ceph/ceph-csi/scripts/install-snapshot.sh delete-crd || true + # install snapshot controller + ${GOPATH}/src/github.com/ceph/ceph-csi/scripts/install-snapshot.sh install +} + +# When an image was built with podman, it needs importing into minikube. +function podman2minikube() +{ + podman image save "${1}" | (eval "$(minikube docker-env --shell bash)" && docker image load) +} + +# Set environment variables +set_env + +# prepare minikube environment +install_minikube + +install_docker +podman2minikube quay.io/cephcsi/cephcsi:canary + +deploy_rook + +# running e2e.test requires librados and librbd +dnf -y install librados2 librbd1 + +# now it is time to run the e2e tests!