Skip to content

Commit 1e5698d

Browse files
k8s-ci-robotk8s-ci-robot
authoredMar 15, 2025··
Merge pull request #2479 from shiftstack/concurrent-e2e-image
ci: build e2e prerequisites in parallel
2 parents 46a69e2 + 752dea6 commit 1e5698d

File tree

4 files changed

+98
-55
lines changed

4 files changed

+98
-55
lines changed
 

‎Makefile

+7-1
Original file line numberDiff line numberDiff line change
@@ -184,7 +184,7 @@ $(E2E_NO_ARTIFACT_TEMPLATES_DIR)/cluster-template.yaml: $(E2E_KUSTOMIZE_DIR)/wit
184184
$(E2E_NO_ARTIFACT_TEMPLATES_DIR)/cluster-template-%.yaml: $(E2E_KUSTOMIZE_DIR)/% $(KUSTOMIZE) FORCE
185185
$(KUSTOMIZE) build "$<" > "$@"
186186

187-
e2e-prerequisites: e2e-templates e2e-image test-e2e-image-prerequisites ## Build all artifacts required by e2e tests
187+
e2e-prerequisites: $(GINKGO) e2e-templates e2e-image test-e2e-image-prerequisites ## Build all artifacts required by e2e tests
188188

189189
# Can be run manually, e.g. via:
190190
# export OPENSTACK_CLOUD_YAML_FILE="$(pwd)/clouds.yaml"
@@ -198,6 +198,12 @@ test-e2e: $(GINKGO) e2e-prerequisites ## Run e2e tests
198198
-config-path="$(E2E_CONF_PATH)" -artifacts-folder="$(ARTIFACTS)" \
199199
-data-folder="$(E2E_DATA_DIR)" $(E2E_ARGS)
200200

201+
# Pre-compile tests
202+
# This is not required, but it will make the tests start faster
203+
.PHONY: build-e2e-tests
204+
build-e2e-tests: $(GINKGO)
205+
$(GINKGO) build -tags=e2e ./test/e2e/suites/e2e/...
206+
201207
.PHONY: e2e-image
202208
e2e-image: CONTROLLER_IMG_TAG = "gcr.io/k8s-staging-capi-openstack/capi-openstack-controller:e2e"
203209
e2e-image: docker-build

‎hack/ci/common.sh

+55
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
#!/usr/bin/env bash
2+
3+
# Copyright 2021 The Kubernetes Authors.
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
# retry $1 times with $2 sleep in between
18+
function retry {
19+
attempt=0
20+
max_attempts=${1}
21+
interval=${2}
22+
shift; shift
23+
until [[ "$attempt" -ge "$max_attempts" ]] ; do
24+
attempt=$((attempt+1))
25+
set +e
26+
eval "$*" && return || echo "failed $attempt times: $*"
27+
set -e
28+
sleep "$interval"
29+
done
30+
echo "error: reached max attempts at retry($*)"
31+
return 1
32+
}
33+
34+
function wait_for_ssh {
35+
local ip=$1 && shift
36+
37+
retry 10 30 "$(get_ssh_cmd) ${ip} -- true"
38+
}
39+
40+
function get_ssh_cmd {
41+
echo "ssh -l cloud $(get_ssh_common_args)"
42+
}
43+
44+
function get_ssh_common_args {
45+
local private_key_file=$(get_ssh_private_key_file)
46+
if [ -z "$private_key_file" ]; then
47+
# If there's no private key file use the public key instead
48+
# This allows us to specify a private key which is held only on a
49+
# hardware device and therefore has no key file
50+
private_key_file=$(get_ssh_public_key_file)
51+
fi
52+
53+
echo "-i ${private_key_file} " \
54+
"-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o IdentitiesOnly=yes -o PasswordAuthentication=no "
55+
}

‎hack/ci/create_devstack.sh

+1-35
Original file line numberDiff line numberDiff line change
@@ -63,22 +63,7 @@ ARTIFACTS=${ARTIFACTS:-/tmp/${CLUSTER_NAME}-artifacts}
6363
devstackdir="${ARTIFACTS}/devstack"
6464
mkdir -p "$devstackdir"
6565

66-
# retry $1 times with $2 sleep in between
67-
function retry {
68-
attempt=0
69-
max_attempts=${1}
70-
interval=${2}
71-
shift; shift
72-
until [[ "$attempt" -ge "$max_attempts" ]] ; do
73-
attempt=$((attempt+1))
74-
set +e
75-
eval "$*" && return || echo "failed $attempt times: $*"
76-
set -e
77-
sleep "$interval"
78-
done
79-
echo "error: reached max attempts at retry($*)"
80-
return 1
81-
}
66+
source "${REPO_ROOT}/hack/ci/common.sh"
8267

8368
function ensure_openstack_client {
8469
if ! command -v openstack;
@@ -102,12 +87,6 @@ function ensure_openstack_client {
10287
fi
10388
}
10489

105-
function wait_for_ssh {
106-
local ip=$1 && shift
107-
108-
retry 10 30 "$(get_ssh_cmd) ${ip} -- true"
109-
}
110-
11190
function start_sshuttle {
11291
if ! command -v sshuttle;
11392
then
@@ -175,19 +154,6 @@ function wait_for_devstack {
175154
done
176155
}
177156

178-
function get_ssh_cmd {
179-
local private_key_file=$(get_ssh_private_key_file)
180-
if [ -z "$private_key_file" ]; then
181-
# If there's no private key file use the public key instead
182-
# This allows us to specify a private key which is held only on a
183-
# hardware device and therefore has no key file
184-
private_key_file=$(get_ssh_public_key_file)
185-
fi
186-
187-
echo "ssh -i ${private_key_file} -l cloud " \
188-
"-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o IdentitiesOnly=yes -o PasswordAuthentication=no "
189-
}
190-
191157
function create_devstack {
192158
local name=$1 && shift
193159
local ip=$1 && shift

‎scripts/ci-e2e.sh

+35-19
Original file line numberDiff line numberDiff line change
@@ -80,25 +80,41 @@ if [ -n "${BOSKOS_HOST:-}" ]; then
8080
HEART_BEAT_PID=$!
8181
fi
8282

83-
"hack/ci/create_devstack.sh"
84-
85-
# Upload image for e2e clusterctl upgrade tests
86-
source "${REPO_ROOT}/hack/ci/${RESOURCE_TYPE}.sh"
87-
CONTAINER_ARCHIVE="/tmp/capo-e2e-image.tar"
88-
SSH_KEY="$(get_ssh_private_key_file)"
89-
SSH_ARGS="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o IdentitiesOnly=yes -o PasswordAuthentication=no"
90-
CONTROLLER_IP=${CONTROLLER_IP:-"10.0.3.15"}
91-
92-
make e2e-image
93-
docker save -o "${CONTAINER_ARCHIVE}" gcr.io/k8s-staging-capi-openstack/capi-openstack-controller:e2e
94-
scp -i "${SSH_KEY}" ${SSH_ARGS} "${CONTAINER_ARCHIVE}" "cloud@${CONTROLLER_IP}:capo-e2e-image.tar"
95-
ssh -i "${SSH_KEY}" ${SSH_ARGS} "cloud@${CONTROLLER_IP}" -- sudo chown root:root capo-e2e-image.tar
96-
ssh -i "${SSH_KEY}" ${SSH_ARGS} "cloud@${CONTROLLER_IP}" -- sudo chmod u=rw,g=r,o=r capo-e2e-image.tar
97-
ssh -i "${SSH_KEY}" ${SSH_ARGS} "cloud@${CONTROLLER_IP}" -- sudo mv capo-e2e-image.tar /var/www/html/capo-e2e-image.tar
98-
99-
export OPENSTACK_CLOUD_YAML_FILE
100-
OPENSTACK_CLOUD_YAML_FILE="$(pwd)/clouds.yaml"
101-
make test-e2e
83+
# Run e2e prerequisites concurrently with devstack build to save time
84+
build_out=$(mktemp)
85+
(
86+
# Run prerequisites at low priority to avoid slowing down devstack tasks,
87+
# which generally take much longer
88+
ionice nice make e2e-image e2e-prerequisites build-e2e-tests
89+
90+
container_archive=$(mktemp --suffix=.tar)
91+
ionice nice docker save -o "${container_archive}" gcr.io/k8s-staging-capi-openstack/capi-openstack-controller:e2e
92+
93+
# Wait for SSH to become available in the provisioning devstack
94+
# infrastructure before uploading image archive
95+
CONTROLLER_IP=${CONTROLLER_IP:-"10.0.3.15"}
96+
source "${REPO_ROOT}/hack/ci/${RESOURCE_TYPE}.sh"
97+
source "${REPO_ROOT}/hack/ci/common.sh"
98+
wait_for_ssh "${CONTROLLER_IP}"
99+
100+
retry 10 10 scp $(get_ssh_common_args) "${container_archive}" "cloud@${CONTROLLER_IP}:capo-e2e-image.tar"
101+
retry 10 10 $(get_ssh_cmd) ${CONTROLLER_IP} -- sudo chown root:root capo-e2e-image.tar
102+
retry 10 10 $(get_ssh_cmd) ${CONTROLLER_IP} -- sudo chmod u=rw,g=r,o=r capo-e2e-image.tar
103+
retry 10 10 $(get_ssh_cmd) ${CONTROLLER_IP} -- sudo mv capo-e2e-image.tar /var/www/html/capo-e2e-image.tar
104+
) >"$build_out" 2>&1 &
105+
build_pid=$!
106+
107+
# Build the devstack environment
108+
hack/ci/create_devstack.sh
109+
110+
# Wait for and capture exit of build process
111+
# Log build output and exit if the build failed
112+
wait $build_pid
113+
ret=$?
114+
cat "$build_out"
115+
[ "$ret" != 0 ] && exit "$ret"
116+
117+
make test-e2e OPENSTACK_CLOUD_YAML_FILE="$(pwd)/clouds.yaml"
102118
test_status="${?}"
103119

104120
# If Boskos is being used then release the resource back to Boskos.

0 commit comments

Comments
 (0)