Skip to content

Commit 1f00bee

Browse files
miz060miz060
authored
Add containerd2 tardev-snapshotter patch (#11934)
1 parent a181899 commit 1f00bee

2 files changed

Lines changed: 294 additions & 1 deletion

File tree

‎SPECS/containerd2/add-tardev-support.patch‎

Lines changed: 288 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,288 @@
1+
From 33ac7f6f513934a0ef59cc6739eee0efdac3631c Mon Sep 17 00:00:00 2001
2+
From: Mitch Zhu <mitchzhu@microsoft.com>
3+
Date: Fri, 22 Nov 2024 20:41:27 +0000
4+
Subject: [PATCH] Enhance snapshot handling and CRI runtime compatibility for
5+
tardev-snapshotter
6+
7+
---
8+
client/image.go | 4 +-
9+
internal/cri/server/container_status_test.go | 2 +-
10+
internal/cri/server/images/image_pull.go | 37 +++++++++++--------
11+
internal/cri/server/podsandbox/controller.go | 2 +-
12+
internal/cri/server/podsandbox/sandbox_run.go | 30 ++++++++-------
13+
internal/cri/server/service.go | 2 +-
14+
internal/cri/store/image/image.go | 29 ++++++++++++---
15+
7 files changed, 68 insertions(+), 38 deletions(-)
16+
17+
diff --git a/client/image.go b/client/image.go
18+
index 355bcba..791db88 100644
19+
--- a/client/image.go
20+
+++ b/client/image.go
21+
@@ -31,6 +31,7 @@ import (
22+
"github.com/containerd/containerd/v2/internal/kmutex"
23+
"github.com/containerd/containerd/v2/pkg/labels"
24+
"github.com/containerd/containerd/v2/pkg/rootfs"
25+
+ "github.com/containerd/containerd/v2/pkg/snapshotters"
26+
"github.com/containerd/errdefs"
27+
"github.com/containerd/platforms"
28+
"github.com/opencontainers/go-digest"
29+
@@ -333,7 +334,8 @@ func (i *image) Unpack(ctx context.Context, snapshotterName string, opts ...Unpa
30+
}
31+
32+
for _, layer := range layers {
33+
- unpacked, err = rootfs.ApplyLayerWithOpts(ctx, layer, chain, sn, a, config.SnapshotOpts, config.ApplyOpts)
34+
+ snOpts := append(config.SnapshotOpts, snapshots.WithLabels(map[string]string{snapshotters.TargetLayerDigestLabel: layer.Blob.Digest.String()}))
35+
+ unpacked, err = rootfs.ApplyLayerWithOpts(ctx, layer, chain, sn, a, snOpts, config.ApplyOpts)
36+
if err != nil {
37+
return fmt.Errorf("apply layer error for %q: %w", i.Name(), err)
38+
}
39+
diff --git a/internal/cri/server/container_status_test.go b/internal/cri/server/container_status_test.go
40+
index 05b1650..71dcc10 100644
41+
--- a/internal/cri/server/container_status_test.go
42+
+++ b/internal/cri/server/container_status_test.go
43+
@@ -302,7 +302,7 @@ func (s *fakeImageService) LocalResolve(refOrID string) (imagestore.Image, error
44+
45+
func (s *fakeImageService) ImageFSPaths() map[string]string { return make(map[string]string) }
46+
47+
-func (s *fakeImageService) PullImage(context.Context, string, func(string) (string, string, error), *runtime.PodSandboxConfig, string) (string, error) {
48+
+func (s *fakeImageService) PullImage(context.Context, string, func(string) (string, string, error), *runtime.PodSandboxConfig, string, string) (string, error) {
49+
return "", errors.New("not implemented")
50+
}
51+
52+
diff --git a/internal/cri/server/images/image_pull.go b/internal/cri/server/images/image_pull.go
53+
index e59b88b..f9c90b7 100644
54+
--- a/internal/cri/server/images/image_pull.go
55+
+++ b/internal/cri/server/images/image_pull.go
56+
@@ -96,6 +96,15 @@ import (
57+
58+
// PullImage pulls an image with authentication config.
59+
func (c *GRPCCRIImageService) PullImage(ctx context.Context, r *runtime.PullImageRequest) (_ *runtime.PullImageResponse, err error) {
60+
+ imageRef := r.GetImage().GetImage()
61+
+ snapshotter, err := c.snapshotterFromPodSandboxConfig(ctx, imageRef, r.SandboxConfig, r.GetImage().GetRuntimeHandler())
62+
+ if err != nil {
63+
+ return nil, err
64+
+ }
65+
+ return c.pullImage(ctx, r, snapshotter)
66+
+}
67+
+
68+
+func (c *GRPCCRIImageService) pullImage(ctx context.Context, r *runtime.PullImageRequest, snapshotter string) (_ *runtime.PullImageResponse, err error) {
69+
70+
imageRef := r.GetImage().GetImage()
71+
72+
@@ -110,14 +119,14 @@ func (c *GRPCCRIImageService) PullImage(ctx context.Context, r *runtime.PullImag
73+
return ParseAuth(hostauth, host)
74+
}
75+
76+
- ref, err := c.CRIImageService.PullImage(ctx, imageRef, credentials, r.SandboxConfig, r.GetImage().GetRuntimeHandler())
77+
+ ref, err := c.CRIImageService.PullImage(ctx, imageRef, credentials, r.SandboxConfig, r.GetImage().GetRuntimeHandler(), snapshotter)
78+
if err != nil {
79+
return nil, err
80+
}
81+
return &runtime.PullImageResponse{ImageRef: ref}, nil
82+
}
83+
84+
-func (c *CRIImageService) PullImage(ctx context.Context, name string, credentials func(string) (string, string, error), sandboxConfig *runtime.PodSandboxConfig, runtimeHandler string) (_ string, err error) {
85+
+func (c *CRIImageService) PullImage(ctx context.Context, name string, credentials func(string) (string, string, error), sandboxConfig *runtime.PodSandboxConfig, runtimeHandler string, snapshotter string) (_ string, err error) {
86+
span := tracing.SpanFromContext(ctx)
87+
defer func() {
88+
// TODO: add domain label for imagePulls metrics, and we may need to provide a mechanism
89+
@@ -167,10 +176,6 @@ func (c *CRIImageService) PullImage(ctx context.Context, name string, credential
90+
)
91+
92+
defer pcancel()
93+
- snapshotter, err := c.snapshotterFromPodSandboxConfig(ctx, ref, sandboxConfig)
94+
- if err != nil {
95+
- return "", err
96+
- }
97+
log.G(ctx).Debugf("PullImage %q with snapshotter %s", ref, snapshotter)
98+
span.SetAttributes(
99+
tracing.Attribute("image.ref", ref),
100+
@@ -761,17 +766,19 @@ func (rt *pullRequestReporterRoundTripper) RoundTrip(req *http.Request) (*http.R
101+
// Once we know the runtime, try to override default snapshotter if it is set for this runtime.
102+
// See https://github.com/containerd/containerd/issues/6657
103+
func (c *CRIImageService) snapshotterFromPodSandboxConfig(ctx context.Context, imageRef string,
104+
- s *runtime.PodSandboxConfig) (string, error) {
105+
+ s *runtime.PodSandboxConfig, runtimeHandler string) (string, error) {
106+
snapshotter := c.config.Snapshotter
107+
- if s == nil || s.Annotations == nil {
108+
- return snapshotter, nil
109+
- }
110+
111+
- // TODO(kiashok): honor the new CRI runtime handler field added to v0.29.0
112+
- // for image pull per runtime class support.
113+
- runtimeHandler, ok := s.Annotations[annotations.RuntimeHandler]
114+
- if !ok {
115+
- return snapshotter, nil
116+
+ if runtimeHandler == "" {
117+
+ if s == nil || s.Annotations == nil {
118+
+ return snapshotter, nil
119+
+ } else {
120+
+ ok := false
121+
+ runtimeHandler, ok = s.Annotations[annotations.RuntimeHandler]
122+
+ if !ok {
123+
+ return snapshotter, nil
124+
+ }
125+
+ }
126+
}
127+
128+
// TODO: Ensure error is returned if runtime not found?
129+
diff --git a/internal/cri/server/podsandbox/controller.go b/internal/cri/server/podsandbox/controller.go
130+
index a185a4c..8fd032b 100644
131+
--- a/internal/cri/server/podsandbox/controller.go
132+
+++ b/internal/cri/server/podsandbox/controller.go
133+
@@ -110,7 +110,7 @@ type RuntimeService interface {
134+
type ImageService interface {
135+
LocalResolve(refOrID string) (imagestore.Image, error)
136+
GetImage(id string) (imagestore.Image, error)
137+
- PullImage(ctx context.Context, name string, creds func(string) (string, string, error), sc *runtime.PodSandboxConfig, runtimeHandler string) (string, error)
138+
+ PullImage(ctx context.Context, name string, creds func(string) (string, string, error), sc *runtime.PodSandboxConfig, runtimeHandler string, snapshotter string) (string, error)
139+
RuntimeSnapshotter(ctx context.Context, ociRuntime criconfig.Runtime) string
140+
PinnedImage(string) string
141+
}
142+
diff --git a/internal/cri/server/podsandbox/sandbox_run.go b/internal/cri/server/podsandbox/sandbox_run.go
143+
index 53d949f..b296061 100644
144+
--- a/internal/cri/server/podsandbox/sandbox_run.go
145+
+++ b/internal/cri/server/podsandbox/sandbox_run.go
146+
@@ -77,23 +77,25 @@ func (c *Controller) Start(ctx context.Context, id string) (cin sandbox.Controll
147+
148+
sandboxImage := c.getSandboxImageName()
149+
// Ensure sandbox container image snapshot.
150+
- image, err := c.ensureImageExists(ctx, sandboxImage, config, metadata.RuntimeHandler)
151+
+ ociRuntime, err := c.config.GetSandboxRuntime(config, metadata.RuntimeHandler)
152+
if err != nil {
153+
- return cin, fmt.Errorf("failed to get sandbox image %q: %w", sandboxImage, err)
154+
+ return cin, fmt.Errorf("failed to get sandbox runtime: %w", err)
155+
}
156+
+ log.G(ctx).WithField("podsandboxid", id).Debugf("use OCI runtime %+v", ociRuntime)
157+
158+
- containerdImage, err := c.toContainerdImage(ctx, *image)
159+
+ labels["oci_runtime_type"] = ociRuntime.Type
160+
+
161+
+ snapshotter := c.imageService.RuntimeSnapshotter(ctx, ociRuntime)
162+
+
163+
+ image, err := c.ensureImageExists(ctx, sandboxImage, config, metadata.RuntimeHandler, snapshotter)
164+
if err != nil {
165+
- return cin, fmt.Errorf("failed to get image from containerd %q: %w", image.ID, err)
166+
+ return cin, fmt.Errorf("failed to get sandbox image %q: %w", sandboxImage, err)
167+
}
168+
169+
- ociRuntime, err := c.config.GetSandboxRuntime(config, metadata.RuntimeHandler)
170+
+ containerdImage, err := c.toContainerdImage(ctx, *image)
171+
if err != nil {
172+
- return cin, fmt.Errorf("failed to get sandbox runtime: %w", err)
173+
+ return cin, fmt.Errorf("failed to get image from containerd %q: %w", image.ID, err)
174+
}
175+
- log.G(ctx).WithField("podsandboxid", id).Debugf("use OCI runtime %+v", ociRuntime)
176+
-
177+
- labels["oci_runtime_type"] = ociRuntime.Type
178+
179+
// Create sandbox container root directories.
180+
sandboxRootDir := c.getSandboxRootDir(id)
181+
@@ -173,7 +175,7 @@ func (c *Controller) Start(ctx context.Context, id string) (cin sandbox.Controll
182+
snapshotterOpt = append(snapshotterOpt, extraSOpts...)
183+
184+
opts := []containerd.NewContainerOpts{
185+
- containerd.WithSnapshotter(c.imageService.RuntimeSnapshotter(ctx, ociRuntime)),
186+
+ containerd.WithSnapshotter(snapshotter),
187+
customopts.WithNewSnapshot(id, containerdImage, snapshotterOpt...),
188+
containerd.WithSpec(spec, specOpts...),
189+
containerd.WithContainerLabels(sandboxLabels),
190+
@@ -299,17 +301,19 @@ func (c *Controller) Create(_ctx context.Context, info sandbox.Sandbox, opts ...
191+
return c.store.Save(podSandbox)
192+
}
193+
194+
-func (c *Controller) ensureImageExists(ctx context.Context, ref string, config *runtime.PodSandboxConfig, runtimeHandler string) (*imagestore.Image, error) {
195+
+func (c *Controller) ensureImageExists(ctx context.Context, ref string, config *runtime.PodSandboxConfig, runtimeHandler string, snapshotter string) (*imagestore.Image, error) {
196+
image, err := c.imageService.LocalResolve(ref)
197+
if err != nil && !errdefs.IsNotFound(err) {
198+
return nil, fmt.Errorf("failed to get image %q: %w", ref, err)
199+
}
200+
if err == nil {
201+
- return &image, nil
202+
+ if _, ok := image.Snapshotters[snapshotter]; ok {
203+
+ return &image, nil
204+
+ }
205+
}
206+
// Pull image to ensure the image exists
207+
// TODO: Cleaner interface
208+
- imageID, err := c.imageService.PullImage(ctx, ref, nil, config, runtimeHandler)
209+
+ imageID, err := c.imageService.PullImage(ctx, ref, nil, config, runtimeHandler, snapshotter)
210+
if err != nil {
211+
return nil, fmt.Errorf("failed to pull image %q: %w", ref, err)
212+
}
213+
diff --git a/internal/cri/server/service.go b/internal/cri/server/service.go
214+
index 37d66f0..5d1546e 100644
215+
--- a/internal/cri/server/service.go
216+
+++ b/internal/cri/server/service.go
217+
@@ -97,7 +97,7 @@ type RuntimeService interface {
218+
type ImageService interface {
219+
RuntimeSnapshotter(ctx context.Context, ociRuntime criconfig.Runtime) string
220+
221+
- PullImage(ctx context.Context, name string, credentials func(string) (string, string, error), sandboxConfig *runtime.PodSandboxConfig, runtimeHandler string) (string, error)
222+
+ PullImage(ctx context.Context, name string, credentials func(string) (string, string, error), sandboxConfig *runtime.PodSandboxConfig, runtimeHandler string, snapshotter string) (string, error)
223+
UpdateImage(ctx context.Context, r string) error
224+
225+
CheckImages(ctx context.Context) error
226+
diff --git a/internal/cri/store/image/image.go b/internal/cri/store/image/image.go
227+
index 5887e75..43ecf0d 100644
228+
--- a/internal/cri/store/image/image.go
229+
+++ b/internal/cri/store/image/image.go
230+
@@ -20,6 +20,7 @@ import (
231+
"context"
232+
"encoding/json"
233+
"fmt"
234+
+ "strings"
235+
"sync"
236+
237+
"github.com/containerd/containerd/v2/core/content"
238+
@@ -53,6 +54,8 @@ type Image struct {
239+
ImageSpec imagespec.Image
240+
// Pinned image to prevent it from garbage collection
241+
Pinned bool
242+
+ // Snapshotters is a map whose keys are snapshotters for which this image has a snapshot.
243+
+ Snapshotters map[string]struct{}
244+
}
245+
246+
// Getter is used to get images but does not make changes
247+
@@ -170,6 +173,19 @@ func (s *Store) getImage(ctx context.Context, i images.Image) (*Image, error) {
248+
return nil, fmt.Errorf("read image config from content store: %w", err)
249+
}
250+
251+
+ info, err := s.provider.Info(ctx, desc.Digest)
252+
+ if err != nil {
253+
+ return nil, fmt.Errorf("get content store config info: %w", err)
254+
+ }
255+
+
256+
+ snapshotters := make(map[string]struct{})
257+
+ for label := range info.Labels {
258+
+ const Prefix = "containerd.io/gc.ref.snapshot."
259+
+ if strings.HasPrefix(label, Prefix) {
260+
+ snapshotters[label[len(Prefix):]] = struct{}{}
261+
+ }
262+
+ }
263+
+
264+
var spec imagespec.Image
265+
if err := json.Unmarshal(blob, &spec); err != nil {
266+
return nil, fmt.Errorf("unmarshal image config %s: %w", blob, err)
267+
@@ -178,12 +194,13 @@ func (s *Store) getImage(ctx context.Context, i images.Image) (*Image, error) {
268+
pinned := i.Labels[labels.PinnedImageLabelKey] == labels.PinnedImageLabelValue
269+
270+
return &Image{
271+
- ID: id,
272+
- References: []string{i.Name},
273+
- ChainID: chainID.String(),
274+
- Size: size,
275+
- ImageSpec: spec,
276+
- Pinned: pinned,
277+
+ ID: id,
278+
+ References: []string{i.Name},
279+
+ ChainID: chainID.String(),
280+
+ Size: size,
281+
+ ImageSpec: spec,
282+
+ Pinned: pinned,
283+
+ Snapshotters: snapshotters,
284+
}, nil
285+
286+
}
287+
--
288+
2.34.1

‎SPECS/containerd2/containerd2.spec‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
Summary: Industry-standard container runtime
66
Name: %{upstream_name}2
77
Version: 2.0.0
8-
Release: 1%{?dist}
8+
Release: 2%{?dist}
99
License: ASL 2.0
1010
Group: Tools/Container
1111
URL: https://www.containerd.io
@@ -15,6 +15,8 @@ Distribution: Azure Linux
1515
Source0: https://github.com/containerd/containerd/archive/v%{version}.tar.gz#/%{upstream_name}-%{version}.tar.gz
1616
Source1: containerd.service
1717
Source2: containerd.toml
18+
# Added patch to support tardev-snapshotter for Kata CC
19+
Patch0: add-tardev-support.patch
1820

1921
%{?systemd_requires}
2022

@@ -79,6 +81,9 @@ fi
7981
%dir /opt/containerd/lib
8082

8183
%changelog
84+
* Thu Jan 09 2024 Mitch Zhu <mitchzhu@microsoft.com> - 2.0.0-2
85+
- Added patch to support tardev-snapshotter for Kata CC.
86+
8287
* Wed Dec 11 2024 Nan Liu <liunan@microsoft.com> - 2.0.0-1
8388
- Created a standalone package for containerd 2.0.0
8489
- Initial CBL-Mariner import from Azure

0 commit comments

Comments
 (0)