Skip to content

Commit 85131c5

Browse files
committed
test: Add test for Transfer Service
Signed-off-by: Kohei Tokunaga <[email protected]>
1 parent 293ceab commit 85131c5

File tree

7 files changed

+104
-25
lines changed

7 files changed

+104
-25
lines changed

.github/workflows/tests.yml

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ jobs:
5656
builtin: ["true", "false"]
5757
metadata-store: ["memory", "db"]
5858
fuse-passthrough: ["true", "false"]
59+
transfer-service: ["true", "false"]
5960
exclude:
6061
- buildargs: ""
6162
builtin: "true"
@@ -69,6 +70,14 @@ jobs:
6970
buildargs: "--build-arg=CONTAINERD_VERSION=main"
7071
- fuse-passthrough: "true"
7172
metadata-store: "db"
73+
- transfer-service: "true"
74+
buildargs: "--build-arg=CONTAINERD_VERSION=main"
75+
- transfer-service: "true"
76+
builtin: "true"
77+
- transfer-service: "true"
78+
metadata-store: "db"
79+
- transfer-service: "true"
80+
fuse-passthrough: "true"
7281
steps:
7382
- name: Install htpasswd for setting up private registry
7483
run: sudo apt-get update -y && sudo apt-get --no-install-recommends install -y apache2-utils
@@ -79,6 +88,7 @@ jobs:
7988
BUILTIN_SNAPSHOTTER: ${{ matrix.builtin }}
8089
METADATA_STORE: ${{ matrix.metadata-store }}
8190
FUSE_PASSTHROUGH: ${{ matrix.fuse-passthrough }}
91+
TRANSFER_SERVICE: ${{ matrix.transfer-service }}
8292
run: make integration
8393

8494
test-optimize:
@@ -149,6 +159,7 @@ jobs:
149159
builtin: ["true", "false"]
150160
metadata-store: ["memory", "db"]
151161
fuse-passthrough: ["true", "false"]
162+
transfer-service: ["true", "false"]
152163
exclude:
153164
- buildargs: ""
154165
builtin: "true"
@@ -162,6 +173,14 @@ jobs:
162173
buildargs: "--build-arg=CONTAINERD_VERSION=main"
163174
- fuse-passthrough: "true"
164175
metadata-store: "db"
176+
- transfer-service: "true"
177+
buildargs: "--build-arg=CONTAINERD_VERSION=main"
178+
- transfer-service: "true"
179+
builtin: "true"
180+
- transfer-service: "true"
181+
metadata-store: "db"
182+
- transfer-service: "true"
183+
fuse-passthrough: "true"
165184
steps:
166185
- uses: actions/checkout@v4
167186
- name: Validate containerd through CRI
@@ -170,6 +189,7 @@ jobs:
170189
BUILTIN_SNAPSHOTTER: ${{ matrix.builtin }}
171190
METADATA_STORE: ${{ matrix.metadata-store }}
172191
FUSE_PASSTHROUGH: ${{ matrix.fuse-passthrough }}
192+
TRANSFER_SERVICE: ${{ matrix.transfer-service }}
173193
run: make test-cri-containerd
174194

175195
test-cri-cri-o:
Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
version = 2
2+
3+
# Basic CRI configuration with enabling Stargz Snapshotter
4+
[plugins."io.containerd.grpc.v1.cri".containerd]
5+
default_runtime_name = "runc"
6+
snapshotter = "stargz"
7+
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
8+
runtime_type = "io.containerd.runc.v2"
9+
10+
# Enable Stargz Snapshotter in Transfer Service
11+
[[plugins."io.containerd.transfer.v1.local".unpack_config]]
12+
platform = "linux"
13+
snapshotter = "stargz"
14+
# Enable overlayfs in Transfer Service
15+
[[plugins."io.containerd.transfer.v1.local".unpack_config]]
16+
platform = "linux"
17+
snapshotter = "overlayfs"
18+
19+
# Plugin Stargz Snapshotter
20+
[proxy_plugins]
21+
[proxy_plugins.stargz]
22+
type = "snapshot"
23+
address = "/run/containerd-stargz-grpc/containerd-stargz-grpc.sock"
24+
[proxy_plugins.stargz.exports]
25+
root = "/var/lib/containerd-stargz-grpc/"
26+
enable_remote_snapshot_annotations = "true"

script/cri-containerd/test-stargz.sh

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -115,11 +115,16 @@ docker exec "${PREPARE_NODE_NAME}" /bin/bash /tools/mirror.sh
115115
# Configure mirror registries for containerd and snapshotter
116116
docker exec "${TEST_NODE_NAME}" cat /etc/containerd/config.toml > "${CONTAINERD_CONFIG}"
117117
docker exec "${TEST_NODE_NAME}" cat /etc/containerd-stargz-grpc/config.toml > "${SNAPSHOTTER_CONFIG}"
118+
docker exec "${TEST_NODE_NAME}" mkdir -p "/etc/containerd/certs.d"
119+
cat <<EOF >> "${CONTAINERD_CONFIG}"
120+
[plugins."io.containerd.cri.v1.images".registry]
121+
config_path = "/etc/containerd/certs.d"
122+
EOF
118123
cat "${IMAGE_LIST}" | sed -E 's/^([^/]*).*/\1/g' | sort | uniq | while read DOMAIN ; do
119124
echo "Adding mirror config: ${DOMAIN}"
120-
cat <<EOF >> "${CONTAINERD_CONFIG}"
121-
[plugins."io.containerd.grpc.v1.cri".registry.mirrors."${DOMAIN}"]
122-
endpoint = ["http://${REGISTRY_HOST}:5000"]
125+
docker exec "${TEST_NODE_NAME}" mkdir -p "/etc/containerd/certs.d/${DOMAIN}/"
126+
cat <<EOF | docker exec -i "${TEST_NODE_NAME}" tee -a "/etc/containerd/certs.d/${DOMAIN}/hosts.toml"
127+
server = "http://${REGISTRY_HOST}:5000"
123128
EOF
124129
if [ "${BUILTIN_SNAPSHOTTER:-}" == "true" ] ; then
125130
cat <<EOF >> "${CONTAINERD_CONFIG}"

script/cri-containerd/test.sh

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ function cleanup {
5050
}
5151
trap 'cleanup "$?"' EXIT SIGHUP SIGINT SIGQUIT SIGTERM
5252

53-
BUILTIN_HACK_INST=
53+
ADDITIONAL_INST=
5454
if [ "${BUILTIN_SNAPSHOTTER:-}" == "true" ] ; then
5555
# Special configuration for CRI containerd + builtin stargz snapshotter
5656
cat <<EOF > "${TMP_CONTEXT}/containerd.hack.toml"
@@ -75,7 +75,7 @@ metadata_store = "memory"
7575
[plugins."io.containerd.snapshotter.v1.stargz".cri_keychain]
7676
enable_keychain = true
7777
EOF
78-
BUILTIN_HACK_INST="COPY containerd.hack.toml /etc/containerd/config.toml"
78+
ADDITIONAL_INST="COPY containerd.hack.toml /etc/containerd/config.toml"
7979
fi
8080

8181
cat <<EOF > "${TMP_CONTEXT}/test.conflist"
@@ -128,6 +128,12 @@ if [ "${FUSE_PASSTHROUGH:-}" != "" ] ; then
128128
fi
129129
fi
130130

131+
if [ "${TRANSFER_SERVICE:-}" == "true" ] ; then
132+
cp "${CONTEXT}/config.containerd.transfer.toml" "${TMP_CONTEXT}/"
133+
ADDITIONAL_INST="${ADDITIONAL_INST}
134+
COPY config.containerd.transfer.toml /etc/containerd/config.toml"
135+
fi
136+
131137
# Prepare the testing node
132138
cat <<EOF > "${TMP_CONTEXT}/Dockerfile"
133139
# Legacy builder that doesn't support TARGETARCH should set this explicitly using --build-arg.
@@ -152,7 +158,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends make && \
152158
153159
COPY ./test.conflist /etc/cni/net.d/test.conflist
154160
155-
${BUILTIN_HACK_INST}
161+
${ADDITIONAL_INST}
156162
157163
RUN if [ "${BUILTIN_SNAPSHOTTER:-}" != "true" ] ; then \
158164
sed -i '1imetadata_store = "${USE_METADATA_STORE}"' "${SNAPSHOTTER_CONFIG_FILE}" && \

script/integration/containerd/config.containerd.toml

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,3 +10,13 @@ check_always = true
1010

1111
[plugins."io.containerd.snapshotter.v1.stargz".registry.mirrors."registry-integration.test"]
1212
endpoint = ["http://registry-alt.test:5000"]
13+
14+
# Enable Stargz Snapshotter in Transfer Service
15+
[[plugins."io.containerd.transfer.v1.local".unpack_config]]
16+
platform = "linux"
17+
snapshotter = "stargz"
18+
19+
# Enable overlayfs in Transfer Service
20+
[[plugins."io.containerd.transfer.v1.local".unpack_config]]
21+
platform = "linux"
22+
snapshotter = "overlayfs"

script/integration/containerd/entrypoint.sh

Lines changed: 25 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -146,13 +146,13 @@ function optimize {
146146
local DST="${2}"
147147
local PLAINHTTP="${3}"
148148
local OPTS=${@:4}
149-
ctr-remote image pull --local -u "${DUMMYUSER}:${DUMMYPASS}" "${SRC}"
149+
ctr-remote image pull -u "${DUMMYUSER}:${DUMMYPASS}" "${SRC}"
150150
ctr-remote image optimize ${OPTS} --oci "${SRC}" "${DST}"
151151
PUSHOPTS=
152152
if [ "${PLAINHTTP}" == "true" ] ; then
153153
PUSHOPTS=--plain-http
154154
fi
155-
ctr-remote image push --local ${PUSHOPTS} -u "${DUMMYUSER}:${DUMMYPASS}" "${DST}"
155+
ctr-remote image push ${PUSHOPTS} -u "${DUMMYUSER}:${DUMMYPASS}" "${DST}"
156156
}
157157

158158
function convert {
@@ -164,17 +164,17 @@ function convert {
164164
if [ "${PLAINHTTP}" == "true" ] ; then
165165
PUSHOPTS=--plain-http
166166
fi
167-
ctr-remote image pull --local -u "${DUMMYUSER}:${DUMMYPASS}" "${SRC}"
167+
ctr-remote image pull -u "${DUMMYUSER}:${DUMMYPASS}" "${SRC}"
168168
ctr-remote image convert ${OPTS} --oci "${SRC}" "${DST}"
169-
ctr-remote image push --local ${PUSHOPTS} -u "${DUMMYUSER}:${DUMMYPASS}" "${DST}"
169+
ctr-remote image push ${PUSHOPTS} -u "${DUMMYUSER}:${DUMMYPASS}" "${DST}"
170170
}
171171

172172
function copy {
173173
local SRC="${1}"
174174
local DST="${2}"
175-
ctr-remote image pull --local --all-platforms "${SRC}"
176-
ctr-remote image tag --local "${SRC}" "${DST}"
177-
ctr-remote image push --local -u "${DUMMYUSER}:${DUMMYPASS}" "${DST}"
175+
ctr-remote image pull --all-platforms "${SRC}"
176+
ctr-remote image tag "${SRC}" "${DST}"
177+
ctr-remote image push -u "${DUMMYUSER}:${DUMMYPASS}" "${DST}"
178178
}
179179

180180
function copy_out_dir {
@@ -192,6 +192,11 @@ function copy_out_dir {
192192
rm "${TMPFILE}"
193193
}
194194

195+
RPULL_COMMAND="rpull"
196+
if [ "${USE_TRANSFER_SERVICE}" == "true" ] ; then
197+
RPULL_COMMAND="pull --snapshotter=stargz"
198+
fi
199+
195200
function dump_dir {
196201
local IMAGE="${1}"
197202
local TARGETDIR="${2}"
@@ -201,9 +206,9 @@ function dump_dir {
201206

202207
reboot_containerd
203208
if [ "${REMOTE}" == "true" ] ; then
204-
run_and_check_remote_snapshots ctr-remote images rpull --user "${DUMMYUSER}:${DUMMYPASS}" "${IMAGE}"
209+
run_and_check_remote_snapshots ctr-remote images ${RPULL_COMMAND} --user "${DUMMYUSER}:${DUMMYPASS}" "${IMAGE}"
205210
else
206-
ctr-remote image pull --local --snapshotter="${SNAPSHOTTER}" --user "${DUMMYUSER}:${DUMMYPASS}" "${IMAGE}"
211+
ctr-remote image pull --snapshotter="${SNAPSHOTTER}" --user "${DUMMYUSER}:${DUMMYPASS}" "${IMAGE}"
207212
fi
208213
copy_out_dir "${IMAGE}" "${TARGETDIR}" "${DEST}" "${SNAPSHOTTER}"
209214
}
@@ -244,6 +249,8 @@ if [ "${BUILTIN_SNAPSHOTTER}" != "true" ] ; then
244249
[proxy_plugins.stargz]
245250
type = "snapshot"
246251
address = "/run/containerd-stargz-grpc/containerd-stargz-grpc.sock"
252+
[proxy_plugins.stargz.exports]
253+
enable_remote_snapshot_annotations = "true"
247254
EOF
248255
fi
249256

@@ -277,9 +284,9 @@ optimize "${REGISTRY_HOST}/alpine:3.15.3" "${REGISTRY_ALT_HOST}:5000/alpine:esgz
277284
# TODO: support external TOC suffix other than "-esgztoc"
278285
optimize "${REGISTRY_HOST}/ubuntu:22.04" "${REGISTRY_HOST}/ubuntu:esgz-50000" "false" --estargz-min-chunk-size=50000
279286
optimize "${REGISTRY_HOST}/ubuntu:22.04" "${REGISTRY_HOST}/ubuntu:esgz-ex" "false" --estargz-external-toc
280-
ctr-remote image push --local -u "${DUMMYUSER}:${DUMMYPASS}" "${REGISTRY_HOST}/ubuntu:esgz-ex-esgztoc"
287+
ctr-remote image push -u "${DUMMYUSER}:${DUMMYPASS}" "${REGISTRY_HOST}/ubuntu:esgz-ex-esgztoc"
281288
convert "${REGISTRY_HOST}/ubuntu:22.04" "${REGISTRY_HOST}/ubuntu:esgz-ex-keep-diff-id" "false" --estargz --estargz-external-toc --estargz-keep-diff-id
282-
ctr-remote image push --local -u "${DUMMYUSER}:${DUMMYPASS}" "${REGISTRY_HOST}/ubuntu:esgz-ex-keep-diff-id-esgztoc"
289+
ctr-remote image push -u "${DUMMYUSER}:${DUMMYPASS}" "${REGISTRY_HOST}/ubuntu:esgz-ex-keep-diff-id-esgztoc"
283290

284291
if [ "${BUILTIN_SNAPSHOTTER}" != "true" ] ; then
285292

@@ -295,21 +302,21 @@ if [ "${BUILTIN_SNAPSHOTTER}" != "true" ] ; then
295302
retry curl -X POST localhost:5001/api/v0/version >/dev/null 2>&1 # wait for up
296303

297304
# stargz snapshotter (default labels)
298-
ctr-remote image pull --local --user "${DUMMYUSER}:${DUMMYPASS}" "${REGISTRY_HOST}/ubuntu:22.04"
305+
ctr-remote image pull --user "${DUMMYUSER}:${DUMMYPASS}" "${REGISTRY_HOST}/ubuntu:22.04"
299306
CID=$(ctr-remote i ipfs-push "${REGISTRY_HOST}/ubuntu:22.04")
300307
reboot_containerd
301308
run_and_check_remote_snapshots ctr-remote i rpull --ipfs "${CID}"
302309
copy_out_dir "${CID}" "/usr" "${USR_STARGZSN_IPFS}" "stargz"
303310

304311
# stargz snapshotter (containerd labels)
305-
ctr-remote image pull --local --user "${DUMMYUSER}:${DUMMYPASS}" "${REGISTRY_HOST}/ubuntu:22.04"
312+
ctr-remote image pull --user "${DUMMYUSER}:${DUMMYPASS}" "${REGISTRY_HOST}/ubuntu:22.04"
306313
CID=$(ctr-remote i ipfs-push "${REGISTRY_HOST}/ubuntu:22.04")
307314
reboot_containerd
308315
run_and_check_remote_snapshots ctr-remote i rpull --use-containerd-labels --ipfs "${CID}"
309316
copy_out_dir "${CID}" "/usr" "${USR_STARGZSN_CTD_IPFS}" "stargz"
310317

311318
# overlayfs snapshotter
312-
ctr-remote image pull --local --user "${DUMMYUSER}:${DUMMYPASS}" "${REGISTRY_HOST}/ubuntu:22.04"
319+
ctr-remote image pull --user "${DUMMYUSER}:${DUMMYPASS}" "${REGISTRY_HOST}/ubuntu:22.04"
313320
CID=$(ctr-remote i ipfs-push --estargz=false "${REGISTRY_HOST}/ubuntu:22.04")
314321
reboot_containerd
315322
ctr-remote i rpull --snapshotter=overlayfs --ipfs "${CID}"
@@ -330,11 +337,11 @@ echo "Testing refreshing and mirror..."
330337

331338
reboot_containerd
332339
echo "Getting image with normal snapshotter..."
333-
ctr-remote image pull --local --user "${DUMMYUSER}:${DUMMYPASS}" "${REGISTRY_HOST}/alpine:esgz"
340+
ctr-remote image pull --user "${DUMMYUSER}:${DUMMYPASS}" "${REGISTRY_HOST}/alpine:esgz"
334341
copy_out_dir "${REGISTRY_HOST}/alpine:esgz" "/usr" "${USR_ORG}" "overlayfs"
335342

336343
echo "Getting image with stargz snapshotter..."
337-
run_and_check_remote_snapshots ctr-remote images rpull --user "${DUMMYUSER}:${DUMMYPASS}" "${REGISTRY_HOST}/alpine:esgz"
344+
run_and_check_remote_snapshots ctr-remote images ${RPULL_COMMAND} --user "${DUMMYUSER}:${DUMMYPASS}" "${REGISTRY_HOST}/alpine:esgz"
338345

339346
REGISTRY_HOST_IP=$(getent hosts "${REGISTRY_HOST}" | awk '{ print $1 }')
340347
REGISTRY_ALT_HOST_IP=$(getent hosts "${REGISTRY_ALT_HOST}" | awk '{ print $1 }')
@@ -474,7 +481,7 @@ diff --no-dereference -qr "${USR_NORMALSN_PLAIN_STARGZ}/" "${USR_STARGZSN_PLAIN_
474481

475482
############
476483
# Try to pull this image from different namespace.
477-
ctr-remote --namespace=dummy images rpull --user "${DUMMYUSER}:${DUMMYPASS}" \
484+
ctr-remote --namespace=dummy images ${RPULL_COMMAND} --user "${DUMMYUSER}:${DUMMYPASS}" \
478485
"${REGISTRY_HOST}/ubuntu:esgz"
479486

480487
############
@@ -489,7 +496,7 @@ mv /etc/containerd-stargz-grpc/config.toml_rm /etc/containerd-stargz-grpc/config
489496
reboot_containerd
490497
if [ "${BUILTIN_SNAPSHOTTER}" != "true" ] ; then
491498
# Snapshots should be available even after restarting the snapshotter with a signal
492-
run_and_check_remote_snapshots ctr-remote images rpull --user "${DUMMYUSER}:${DUMMYPASS}" "${REGISTRY_HOST}/alpine:esgz"
499+
run_and_check_remote_snapshots ctr-remote images ${RPULL_COMMAND} --user "${DUMMYUSER}:${DUMMYPASS}" "${REGISTRY_HOST}/alpine:esgz"
493500
ctr-remote run --rm --snapshotter=stargz "${REGISTRY_HOST}/alpine:esgz" test echo hi
494501

495502
TARGET_SIGNALS=(SIGINT SIGTERM)

script/integration/test.sh

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,11 @@ if [ "${FUSE_PASSTHROUGH:-}" != "" ] ; then
6969
fi
7070
fi
7171

72+
USE_TRANSFER_SERVICE=false
73+
if [ "${TRANSFER_SERVICE:-}" == "true" ] ; then
74+
USE_TRANSFER_SERVICE=true
75+
fi
76+
7277
DOCKER_COMPOSE_YAML=$(mktemp)
7378
AUTH_DIR=$(mktemp -d)
7479
SS_ROOT_DIR=$(mktemp -d)
@@ -107,7 +112,7 @@ RUN if [ "${BUILTIN_SNAPSHOTTER:-}" != "true" ] ; then \
107112
fi
108113
109114
ENV CONTAINERD_SNAPSHOTTER=""
110-
115+
ENV USE_TRANSFER_SERVICE="${USE_TRANSFER_SERVICE}"
111116
ENTRYPOINT [ "/entrypoint.sh" ]
112117
EOF
113118
docker build ${DOCKER_BUILD_ARGS:-} -t "${INTEGRATION_TEST_IMAGE_NAME}" ${DOCKER_BUILD_ARGS:-} "${TMP_CONTEXT}"

0 commit comments

Comments
 (0)