Skip to content

Commit fd445ac

Browse files
authored
Merge pull request #2084 from ktock/transferservice
Add docs and tests for Transfer Service
2 parents 1b3ba86 + b405072 commit fd445ac

File tree

9 files changed

+204
-26
lines changed

9 files changed

+204
-26
lines changed

.github/workflows/tests.yml

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ jobs:
5656
builtin: ["true", "false"]
5757
metadata-store: ["memory", "db"]
5858
fuse-passthrough: ["true", "false"]
59+
transfer-service: ["true", "false"]
5960
exclude:
6061
- buildargs: ""
6162
builtin: "true"
@@ -69,6 +70,14 @@ jobs:
6970
buildargs: "--build-arg=CONTAINERD_VERSION=main"
7071
- fuse-passthrough: "true"
7172
metadata-store: "db"
73+
- transfer-service: "true"
74+
buildargs: "--build-arg=CONTAINERD_VERSION=main"
75+
- transfer-service: "true"
76+
builtin: "true"
77+
- transfer-service: "true"
78+
metadata-store: "db"
79+
- transfer-service: "true"
80+
fuse-passthrough: "true"
7281
steps:
7382
- name: Install htpasswd for setting up private registry
7483
run: sudo apt-get update -y && sudo apt-get --no-install-recommends install -y apache2-utils
@@ -79,6 +88,7 @@ jobs:
7988
BUILTIN_SNAPSHOTTER: ${{ matrix.builtin }}
8089
METADATA_STORE: ${{ matrix.metadata-store }}
8190
FUSE_PASSTHROUGH: ${{ matrix.fuse-passthrough }}
91+
TRANSFER_SERVICE: ${{ matrix.transfer-service }}
8292
run: make integration
8393

8494
test-optimize:
@@ -149,6 +159,7 @@ jobs:
149159
builtin: ["true", "false"]
150160
metadata-store: ["memory", "db"]
151161
fuse-passthrough: ["true", "false"]
162+
transfer-service: ["true", "false"]
152163
exclude:
153164
- buildargs: ""
154165
builtin: "true"
@@ -162,6 +173,14 @@ jobs:
162173
buildargs: "--build-arg=CONTAINERD_VERSION=main"
163174
- fuse-passthrough: "true"
164175
metadata-store: "db"
176+
- transfer-service: "true"
177+
buildargs: "--build-arg=CONTAINERD_VERSION=main"
178+
- transfer-service: "true"
179+
builtin: "true"
180+
- transfer-service: "true"
181+
metadata-store: "db"
182+
- transfer-service: "true"
183+
fuse-passthrough: "true"
165184
steps:
166185
- uses: actions/checkout@v4
167186
- name: Validate containerd through CRI
@@ -170,6 +189,7 @@ jobs:
170189
BUILTIN_SNAPSHOTTER: ${{ matrix.builtin }}
171190
METADATA_STORE: ${{ matrix.metadata-store }}
172191
FUSE_PASSTHROUGH: ${{ matrix.fuse-passthrough }}
192+
TRANSFER_SERVICE: ${{ matrix.transfer-service }}
173193
run: make test-cri-containerd
174194

175195
test-cri-cri-o:

Dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ ARG NETAVARK_VERSION=v1.13.0
2727

2828
ARG CONTAINERIZED_SYSTEMD_VERSION=v0.1.1
2929
ARG SLIRP4NETNS_VERSION=v1.3.1
30-
ARG PAUSE_IMAGE_NAME_TEST=registry.k8s.io/pause:3.10
30+
ARG PAUSE_IMAGE_NAME_TEST=registry.k8s.io/pause:3.10.1
3131

3232
# Used in CI
3333
ARG CRI_TOOLS_VERSION=v1.30.1

docs/transfer.md

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
# Enabling Stargz Snapshotter With Transfer Service
2+
3+
Transfer Service is a containerd component which is used for image management in contianerd (e.g. pulling and pushing images).
4+
For details about Transfer Service, refer to [the official document in the containerd repo](https://github.com/containerd/containerd/blob/6af7c07905a317d4c343a49255e2392f4c8569f9/docs/transfer.md).
5+
6+
To use Stargz Snapshotter on containerd with enabling Transfer Service, additional configurations is needed.
7+
8+
## Availability of Transfer Service
9+
10+
Transfer Service is available since v1.7.
11+
And this is enabled in different settings depending on the containerd version.
12+
13+
|containerd version|`ctr`|CRI|
14+
---|---|---
15+
|containerd >= v1.7 and < v2.0|Disabled by default. Enabled by `--local=false`|Disabled|
16+
|containerd >= v2.0 and < v2.1|Enabled by default. Disabled by `--local`|Disabled|
17+
|containerd >= v2.1|Enabled by default. Disabled by `--local`|Enabled by default. Disabled when conditions described in [containerd's CRI document](https://github.com/containerd/containerd/blob/v2.1.0/docs/cri/config.md#image-pull-configuration-since-containerd-v21) are met|
18+
19+
### Note about containerd v2.1
20+
21+
Before containerd v2.1, `disable_snapshot_annotations = false` in containerd's config TOML was a mandatory field to enable Stargz Snapshotter in CRI.
22+
In containerd v2.1, `disable_snapshot_annotations = false` field can still be used to enable Stargz Snapshotter and containerd disables Transfer Service when this field is detected.
23+
If you want to enable Transfer Service, you need to remove `disable_snapshot_annotations = false` field and apply the configuration explaind in this document.
24+
25+
## How to enable Stargz Snapshotter when Transfer Service is enabled?
26+
27+
In containerd v2.1, Transfer Service added support for remote snapshotters like Stargz Snapshotter.
28+
29+
### For ctr and other non-CRI clients
30+
31+
To enable Stargz Snapshotter with Transfer Service, you need to start containerd-stargz-grpc on the node and add the following configuration to contianerd's config TOML file.
32+
Note that you need to add a field `enable_remote_snapshot_annotations = "true"` in `proxy_plugins.stargz.exports` so that containerd can correctly pass image-related information to Stargz Snapshotter.
33+
34+
```toml
35+
version = 2
36+
37+
# Enable Stargz Snapshotter in Transfer Service
38+
[[plugins."io.containerd.transfer.v1.local".unpack_config]]
39+
platform = "linux"
40+
snapshotter = "stargz"
41+
42+
# Plugin Stargz Snapshotter
43+
[proxy_plugins]
44+
[proxy_plugins.stargz]
45+
type = "snapshot"
46+
address = "/run/containerd-stargz-grpc/containerd-stargz-grpc.sock"
47+
[proxy_plugins.stargz.exports]
48+
root = "/var/lib/containerd-stargz-grpc/"
49+
enable_remote_snapshot_annotations = "true"
50+
```
51+
52+
#### Example client command
53+
54+
When you enable Transfer Service with Stargz Snapshotter, you can perform lazy pulling using the normal `ctr` command. (of course, `ctr-remote` can still be used)
55+
56+
```
57+
# ctr image pull --snapshotter=stargz ghcr.io/stargz-containers/ubuntu:24.04-esgz
58+
```
59+
60+
Then `mount | grep stargz` prints stargz mounts on the node.
61+
62+
### For CRI
63+
64+
To enable Stargz Snapshotter with Transfer Service, you need to start containerd-stargz-grpc on the node and add the following configuration to contianerd's config TOML file.
65+
66+
```toml
67+
version = 2
68+
69+
# Basic CRI configuration with enabling Stargz Snapshotter
70+
[plugins."io.containerd.grpc.v1.cri".containerd]
71+
default_runtime_name = "runc"
72+
snapshotter = "stargz"
73+
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
74+
runtime_type = "io.containerd.runc.v2"
75+
76+
# Enable Stargz Snapshotter in Transfer Service
77+
[[plugins."io.containerd.transfer.v1.local".unpack_config]]
78+
platform = "linux"
79+
snapshotter = "stargz"
80+
81+
# Plugin Stargz Snapshotter
82+
[proxy_plugins]
83+
[proxy_plugins.stargz]
84+
type = "snapshot"
85+
address = "/run/containerd-stargz-grpc/containerd-stargz-grpc.sock"
86+
[proxy_plugins.stargz.exports]
87+
root = "/var/lib/containerd-stargz-grpc/"
88+
enable_remote_snapshot_annotations = "true"
89+
```
90+
91+
#### Example client command
92+
93+
You can quickly check the behaviour using `crictl` command.
94+
95+
```
96+
# crictl image pull ghcr.io/stargz-containers/ubuntu:24.04-esgz
97+
```
98+
99+
Then `mount | grep stargz` prints stargz mounts on the node.
Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
version = 2
2+
3+
# Basic CRI configuration with enabling Stargz Snapshotter
4+
[plugins."io.containerd.grpc.v1.cri".containerd]
5+
default_runtime_name = "runc"
6+
snapshotter = "stargz"
7+
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
8+
runtime_type = "io.containerd.runc.v2"
9+
10+
# Enable Stargz Snapshotter in Transfer Service
11+
[[plugins."io.containerd.transfer.v1.local".unpack_config]]
12+
platform = "linux"
13+
snapshotter = "stargz"
14+
# Enable overlayfs in Transfer Service
15+
[[plugins."io.containerd.transfer.v1.local".unpack_config]]
16+
platform = "linux"
17+
snapshotter = "overlayfs"
18+
19+
# Plugin Stargz Snapshotter
20+
[proxy_plugins]
21+
[proxy_plugins.stargz]
22+
type = "snapshot"
23+
address = "/run/containerd-stargz-grpc/containerd-stargz-grpc.sock"
24+
[proxy_plugins.stargz.exports]
25+
root = "/var/lib/containerd-stargz-grpc/"
26+
enable_remote_snapshot_annotations = "true"

script/cri-containerd/test-stargz.sh

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -115,11 +115,16 @@ docker exec "${PREPARE_NODE_NAME}" /bin/bash /tools/mirror.sh
115115
# Configure mirror registries for containerd and snapshotter
116116
docker exec "${TEST_NODE_NAME}" cat /etc/containerd/config.toml > "${CONTAINERD_CONFIG}"
117117
docker exec "${TEST_NODE_NAME}" cat /etc/containerd-stargz-grpc/config.toml > "${SNAPSHOTTER_CONFIG}"
118+
docker exec "${TEST_NODE_NAME}" mkdir -p "/etc/containerd/certs.d"
119+
cat <<EOF >> "${CONTAINERD_CONFIG}"
120+
[plugins."io.containerd.cri.v1.images".registry]
121+
config_path = "/etc/containerd/certs.d"
122+
EOF
118123
cat "${IMAGE_LIST}" | sed -E 's/^([^/]*).*/\1/g' | sort | uniq | while read DOMAIN ; do
119124
echo "Adding mirror config: ${DOMAIN}"
120-
cat <<EOF >> "${CONTAINERD_CONFIG}"
121-
[plugins."io.containerd.grpc.v1.cri".registry.mirrors."${DOMAIN}"]
122-
endpoint = ["http://${REGISTRY_HOST}:5000"]
125+
docker exec "${TEST_NODE_NAME}" mkdir -p "/etc/containerd/certs.d/${DOMAIN}/"
126+
cat <<EOF | docker exec -i "${TEST_NODE_NAME}" tee -a "/etc/containerd/certs.d/${DOMAIN}/hosts.toml"
127+
server = "http://${REGISTRY_HOST}:5000"
123128
EOF
124129
if [ "${BUILTIN_SNAPSHOTTER:-}" == "true" ] ; then
125130
cat <<EOF >> "${CONTAINERD_CONFIG}"

script/cri-containerd/test.sh

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ function cleanup {
5050
}
5151
trap 'cleanup "$?"' EXIT SIGHUP SIGINT SIGQUIT SIGTERM
5252

53-
BUILTIN_HACK_INST=
53+
ADDITIONAL_INST=
5454
if [ "${BUILTIN_SNAPSHOTTER:-}" == "true" ] ; then
5555
# Special configuration for CRI containerd + builtin stargz snapshotter
5656
cat <<EOF > "${TMP_CONTEXT}/containerd.hack.toml"
@@ -75,7 +75,7 @@ metadata_store = "memory"
7575
[plugins."io.containerd.snapshotter.v1.stargz".cri_keychain]
7676
enable_keychain = true
7777
EOF
78-
BUILTIN_HACK_INST="COPY containerd.hack.toml /etc/containerd/config.toml"
78+
ADDITIONAL_INST="COPY containerd.hack.toml /etc/containerd/config.toml"
7979
fi
8080

8181
cat <<EOF > "${TMP_CONTEXT}/test.conflist"
@@ -128,6 +128,12 @@ if [ "${FUSE_PASSTHROUGH:-}" != "" ] ; then
128128
fi
129129
fi
130130

131+
if [ "${TRANSFER_SERVICE:-}" == "true" ] ; then
132+
cp "${CONTEXT}/config.containerd.transfer.toml" "${TMP_CONTEXT}/"
133+
ADDITIONAL_INST="${ADDITIONAL_INST}
134+
COPY config.containerd.transfer.toml /etc/containerd/config.toml"
135+
fi
136+
131137
# Prepare the testing node
132138
cat <<EOF > "${TMP_CONTEXT}/Dockerfile"
133139
# Legacy builder that doesn't support TARGETARCH should set this explicitly using --build-arg.
@@ -152,7 +158,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends make && \
152158
153159
COPY ./test.conflist /etc/cni/net.d/test.conflist
154160
155-
${BUILTIN_HACK_INST}
161+
${ADDITIONAL_INST}
156162
157163
RUN if [ "${BUILTIN_SNAPSHOTTER:-}" != "true" ] ; then \
158164
sed -i '1imetadata_store = "${USE_METADATA_STORE}"' "${SNAPSHOTTER_CONFIG_FILE}" && \

script/integration/containerd/config.containerd.toml

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,3 +10,13 @@ check_always = true
1010

1111
[plugins."io.containerd.snapshotter.v1.stargz".registry.mirrors."registry-integration.test"]
1212
endpoint = ["http://registry-alt.test:5000"]
13+
14+
# Enable Stargz Snapshotter in Transfer Service
15+
[[plugins."io.containerd.transfer.v1.local".unpack_config]]
16+
platform = "linux"
17+
snapshotter = "stargz"
18+
19+
# Enable overlayfs in Transfer Service
20+
[[plugins."io.containerd.transfer.v1.local".unpack_config]]
21+
platform = "linux"
22+
snapshotter = "overlayfs"

0 commit comments

Comments
 (0)