Skip to content

Commit 8892911

Browse files
author
Kazuyoshi Kato
authored
Merge pull request #623 from kzys/broken-rootfs
Reduce the number of "broken" rootfs files
2 parents 66f4d23 + d7f6fb3 commit 8892911

File tree

4 files changed

+57
-36
lines changed

4 files changed

+57
-36
lines changed

‎Makefile‎

Lines changed: 3 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -227,16 +227,8 @@ DEFAULT_RUNC_JAILER_CONFIG_INSTALLPATH?=/etc/containerd/firecracker-runc-config.
227227
$(DEFAULT_RUNC_JAILER_CONFIG_INSTALLPATH): $(ETC_CONTAINERD) runtime/firecracker-runc-config.json.example
228228
install -D -o root -g root -m400 runtime/firecracker-runc-config.json.example $@
229229

230-
ROOTFS_SLOW_BOOT_INSTALLPATH=$(FIRECRACKER_CONTAINERD_RUNTIME_DIR)/rootfs-slow-boot.img
231-
$(ROOTFS_SLOW_BOOT_INSTALLPATH): tools/image-builder/rootfs-slow-boot.img $(FIRECRACKER_CONTAINERD_RUNTIME_DIR)
232-
install -D -o root -g root -m400 $< $@
233-
234-
ROOTFS_SLOW_REBOOT_INSTALLPATH=$(FIRECRACKER_CONTAINERD_RUNTIME_DIR)/rootfs-slow-reboot.img
235-
$(ROOTFS_SLOW_REBOOT_INSTALLPATH): tools/image-builder/rootfs-slow-reboot.img $(FIRECRACKER_CONTAINERD_RUNTIME_DIR)
236-
install -D -o root -g root -m400 $< $@
237-
238-
ROOTFS_NO_AGENT_INSTALLPATH=$(FIRECRACKER_CONTAINERD_RUNTIME_DIR)/rootfs-no-agent.img
239-
$(ROOTFS_NO_AGENT_INSTALLPATH): tools/image-builder/rootfs-no-agent.img $(FIRECRACKER_CONTAINERD_RUNTIME_DIR)
230+
ROOTFS_DEBUG_INSTALLPATH=$(FIRECRACKER_CONTAINERD_RUNTIME_DIR)/rootfs-debug.img
231+
$(ROOTFS_DEBUG_INSTALLPATH): tools/image-builder/rootfs-debug.img $(FIRECRACKER_CONTAINERD_RUNTIME_DIR)
240232
install -D -o root -g root -m400 $< $@
241233

242234
ROOTFS_STARGZ_INSTALLPATH=$(FIRECRACKER_CONTAINERD_RUNTIME_DIR)/rootfs-stargz.img
@@ -256,7 +248,7 @@ install-default-rootfs: $(DEFAULT_ROOTFS_INSTALLPATH)
256248
install-default-runc-jailer-config: $(DEFAULT_RUNC_JAILER_CONFIG_INSTALLPATH)
257249

258250
.PHONY: install-test-rootfs
259-
install-test-rootfs: $(ROOTFS_SLOW_BOOT_INSTALLPATH) $(ROOTFS_SLOW_REBOOT_INSTALLPATH) $(ROOTFS_NO_AGENT_INSTALLPATH)
251+
install-test-rootfs: $(ROOTFS_DEBUG_INSTALLPATH)
260252

261253
.PHONY: install-stargz-rootfs
262254
install-stargz-rootfs: $(ROOTFS_STARGZ_INSTALLPATH)

‎runtime/service_integ_test.go‎

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1631,6 +1631,8 @@ func TestStopVM_Isolated(t *testing.T) {
16311631
image, err := alpineImage(ctx, client, defaultSnapshotterName)
16321632
require.NoError(err, "failed to get alpine image")
16331633

1634+
kernelArgs := integtest.DefaultRuntimeConfig.KernelArgs
1635+
16341636
tests := []struct {
16351637
name string
16361638
createVMRequest proto.CreateVMRequest
@@ -1656,8 +1658,9 @@ func TestStopVM_Isolated(t *testing.T) {
16561658
withStopVM: true,
16571659

16581660
createVMRequest: proto.CreateVMRequest{
1661+
KernelArgs: kernelArgs + " failure=slow-reboot",
16591662
RootDrive: &proto.FirecrackerRootDrive{
1660-
HostPath: "/var/lib/firecracker-containerd/runtime/rootfs-slow-reboot.img",
1663+
HostPath: "/var/lib/firecracker-containerd/runtime/rootfs-debug.img",
16611664
},
16621665
},
16631666
stopFunc: func(ctx context.Context, fcClient fccontrol.FirecrackerService, req proto.CreateVMRequest) {
@@ -1692,8 +1695,9 @@ func TestStopVM_Isolated(t *testing.T) {
16921695
withStopVM: true,
16931696

16941697
createVMRequest: proto.CreateVMRequest{
1698+
KernelArgs: kernelArgs + " failure=slow-reboot",
16951699
RootDrive: &proto.FirecrackerRootDrive{
1696-
HostPath: "/var/lib/firecracker-containerd/runtime/rootfs-slow-reboot.img",
1700+
HostPath: "/var/lib/firecracker-containerd/runtime/rootfs-debug.img",
16971701
},
16981702
},
16991703
stopFunc: func(ctx context.Context, fcClient fccontrol.FirecrackerService, req proto.CreateVMRequest) {
@@ -2137,6 +2141,8 @@ func TestCreateVM_Isolated(t *testing.T) {
21372141
fcClient, err := newFCControlClient(containerdSockPath)
21382142
require.NoError(t, err, "failed to create ttrpc client")
21392143

2144+
kernelArgs := integtest.DefaultRuntimeConfig.KernelArgs
2145+
21402146
type subtest struct {
21412147
name string
21422148
request proto.CreateVMRequest
@@ -2155,11 +2161,12 @@ func TestCreateVM_Isolated(t *testing.T) {
21552161
stopVM: true,
21562162
},
21572163
{
2158-
name: "Error Case",
2164+
name: "No Agent",
21592165
request: proto.CreateVMRequest{
21602166
TimeoutSeconds: 5,
2167+
KernelArgs: kernelArgs + " failure=no-agent",
21612168
RootDrive: &proto.FirecrackerRootDrive{
2162-
HostPath: "/var/lib/firecracker-containerd/runtime/rootfs-no-agent.img",
2169+
HostPath: "/var/lib/firecracker-containerd/runtime/rootfs-debug.img",
21632170
},
21642171
},
21652172
validate: func(t *testing.T, err error) {
@@ -2175,8 +2182,9 @@ func TestCreateVM_Isolated(t *testing.T) {
21752182
{
21762183
name: "Slow Root FS",
21772184
request: proto.CreateVMRequest{
2185+
KernelArgs: kernelArgs + " failure=slow-boot",
21782186
RootDrive: &proto.FirecrackerRootDrive{
2179-
HostPath: "/var/lib/firecracker-containerd/runtime/rootfs-slow-boot.img",
2187+
HostPath: "/var/lib/firecracker-containerd/runtime/rootfs-debug.img",
21802188
},
21812189
},
21822190
validate: func(t *testing.T, err error) {
@@ -2189,8 +2197,9 @@ func TestCreateVM_Isolated(t *testing.T) {
21892197
{
21902198
name: "Slow Root FS and Longer Timeout",
21912199
request: proto.CreateVMRequest{
2200+
KernelArgs: kernelArgs + " failure=slow-boot",
21922201
RootDrive: &proto.FirecrackerRootDrive{
2193-
HostPath: "/var/lib/firecracker-containerd/runtime/rootfs-slow-boot.img",
2202+
HostPath: "/var/lib/firecracker-containerd/runtime/rootfs-debug.img",
21942203
},
21952204
TimeoutSeconds: 60,
21962205
},

‎tools/image-builder/Makefile‎

Lines changed: 6 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ define run_debootstrap =
5555
touch $@
5656
endef
5757

58-
all: rootfs.img rootfs-slow-boot.img rootfs-slow-reboot.img rootfs-no-agent.img
58+
all: rootfs.img rootfs-debug.img
5959
stargz: rootfs-stargz.img
6060

6161
$(WORKDIR):
@@ -89,25 +89,12 @@ endif
8989
rootfs.img: files_common_stamp files_debootstrap_stamp files_ephemeral_stamp
9090
mksquashfs "$(WORKDIR)" rootfs.img -noappend
9191

92-
# Intentionally break the rootfs to simulate the case where CreateVM is taking longer than the minimal timeout
93-
rootfs-slow-boot.img: files_common_stamp files_debootstrap_stamp files_ephemeral_stamp
92+
rootfs-debug.img: files_common_stamp files_debootstrap_stamp files_ephemeral_stamp
9493
rm -fr tmp/$@
9594
cp -a "$(WORKDIR)" tmp/$@
96-
echo 'ExecStartPre=/bin/sleep 30' >> tmp/$@/etc/systemd/system/firecracker-agent.service
97-
mksquashfs tmp/$@ $@ -noappend
98-
99-
# Intentionally break the rootfs to simulate the case where StopVM is taking longer than the minimal timeout
100-
rootfs-slow-reboot.img: files_common_stamp files_debootstrap_stamp files_ephemeral_stamp
101-
rm -fr tmp/$@
102-
cp -a "$(WORKDIR)" tmp/$@
103-
echo 'ExecStop=/bin/sleep 60' >> tmp/$@/etc/systemd/system/firecracker-agent.service
104-
mksquashfs tmp/$@ $@ -noappend
105-
106-
rootfs-no-agent.img: files_common_stamp files_debootstrap_stamp files_ephemeral_stamp
107-
rm -fr tmp/$@
108-
cp -a "$(WORKDIR)" tmp/$@
109-
rm tmp/$@/etc/systemd/system/firecracker-agent.service
110-
rm tmp/$@/usr/local/bin/agent
95+
mv tmp/$@/usr/local/bin/agent tmp/$@/usr/local/bin/agent.real
96+
cp agent.sh tmp/$@/usr/local/bin/agent
97+
ls -l tmp/$@/usr/local/bin
11198
mksquashfs tmp/$@ $@ -noappend
11299

113100
rootfs-stargz.img: WORKDIR := $(addsuffix -stargz, $(WORKDIR))
@@ -141,7 +128,7 @@ builder_stamp:
141128
clean:
142129
-rm -f *stamp
143130
if [ $(UID) -eq 0 ]; then \
144-
rm -f rootfs.img rootfs-slow-boot.img rootfs-slow-reboot.img rootfs-stargz.img;\
131+
rm -f rootfs.img rootfs-debug.img rootfs-stargz.img;\
145132
else \
146133
$(MAKE) clean-in-docker ;\
147134
fi

‎tools/image-builder/agent.sh‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
#! /bin/bash
2+
# This script adds multiple failure modes to firecracker-containerd's in-VM agent.
3+
set -euo pipefail
4+
5+
agent=/usr/local/bin/agent.real
6+
7+
if [[ "$(cat /proc/cmdline)" =~ failure=([a-z-]+) ]];
8+
then
9+
failure="${BASH_REMATCH[1]}"
10+
else
11+
failure='none'
12+
fi
13+
14+
case "$failure" in
15+
slow-boot)
16+
sleep 30
17+
$agent
18+
;;
19+
slow-reboot)
20+
$agent
21+
sleep 30
22+
;;
23+
no-agent)
24+
exit 1
25+
;;
26+
none)
27+
$agent
28+
;;
29+
*)
30+
echo "unknown failure mode: $failure"
31+
exit 1
32+
esac
33+

0 commit comments

Comments
 (0)