Reduce the number of "broken" rootfs files

There are multiple rootfs files to simulate multiple failure cases.
However building a rootfs image is a slow operation in our build
process and affects our productivity.

This commit consolidates the "broken" rootfs files into one and
use Linux's boot parameter to simulate multiple failure cases.

Signed-off-by: Kazuyoshi Kato <katokazu@amazon.com>

Author: Kazuyoshi Kato

Makefile

@@ -226,16 +226,8 @@ DEFAULT_RUNC_JAILER_CONFIG_INSTALLPATH?=/etc/containerd/firecracker-runc-config.
 $(DEFAULT_RUNC_JAILER_CONFIG_INSTALLPATH): $(ETC_CONTAINERD) runtime/firecracker-runc-config.json.example
 	install -D -o root -g root -m400 runtime/firecracker-runc-config.json.example $@
 
-ROOTFS_SLOW_BOOT_INSTALLPATH=$(FIRECRACKER_CONTAINERD_RUNTIME_DIR)/rootfs-slow-boot.img
-$(ROOTFS_SLOW_BOOT_INSTALLPATH): tools/image-builder/rootfs-slow-boot.img $(FIRECRACKER_CONTAINERD_RUNTIME_DIR)
-	install -D -o root -g root -m400 $< $@
-
-ROOTFS_SLOW_REBOOT_INSTALLPATH=$(FIRECRACKER_CONTAINERD_RUNTIME_DIR)/rootfs-slow-reboot.img
-$(ROOTFS_SLOW_REBOOT_INSTALLPATH): tools/image-builder/rootfs-slow-reboot.img $(FIRECRACKER_CONTAINERD_RUNTIME_DIR)
-	install -D -o root -g root -m400 $< $@
-
-ROOTFS_NO_AGENT_INSTALLPATH=$(FIRECRACKER_CONTAINERD_RUNTIME_DIR)/rootfs-no-agent.img
-$(ROOTFS_NO_AGENT_INSTALLPATH): tools/image-builder/rootfs-no-agent.img $(FIRECRACKER_CONTAINERD_RUNTIME_DIR)
+ROOTFS_DEBUG_INSTALLPATH=$(FIRECRACKER_CONTAINERD_RUNTIME_DIR)/rootfs-debug.img
+$(ROOTFS_DEBUG_INSTALLPATH): tools/image-builder/rootfs-debug.img $(FIRECRACKER_CONTAINERD_RUNTIME_DIR)
 	install -D -o root -g root -m400 $< $@
 
 ROOTFS_STARGZ_INSTALLPATH=$(FIRECRACKER_CONTAINERD_RUNTIME_DIR)/rootfs-stargz.img
@@ -255,7 +247,7 @@ install-default-rootfs: $(DEFAULT_ROOTFS_INSTALLPATH)
 install-default-runc-jailer-config: $(DEFAULT_RUNC_JAILER_CONFIG_INSTALLPATH)
 
 .PHONY: install-test-rootfs
-install-test-rootfs: $(ROOTFS_SLOW_BOOT_INSTALLPATH) $(ROOTFS_SLOW_REBOOT_INSTALLPATH) $(ROOTFS_NO_AGENT_INSTALLPATH)
+install-test-rootfs: $(ROOTFS_DEBUG_INSTALLPATH)
 
 .PHONY: install-stargz-rootfs
 install-stargz-rootfs: $(ROOTFS_STARGZ_INSTALLPATH)

runtime/service_integ_test.go

@@ -1631,6 +1631,8 @@ func TestStopVM_Isolated(t *testing.T) {
 	image, err := alpineImage(ctx, client, defaultSnapshotterName)
 	require.NoError(err, "failed to get alpine image")
 
+	kernelArgs := integtest.DefaultRuntimeConfig.KernelArgs
+
 	tests := []struct {
 		name            string
 		createVMRequest proto.CreateVMRequest
@@ -1656,8 +1658,9 @@ func TestStopVM_Isolated(t *testing.T) {
 			withStopVM: true,
 
 			createVMRequest: proto.CreateVMRequest{
+				KernelArgs: kernelArgs + " failure=slow-reboot",
 				RootDrive: &proto.FirecrackerRootDrive{
-					HostPath: "/var/lib/firecracker-containerd/runtime/rootfs-slow-reboot.img",
+					HostPath: "/var/lib/firecracker-containerd/runtime/rootfs-debug.img",
 				},
 			},
 			stopFunc: func(ctx context.Context, fcClient fccontrol.FirecrackerService, req proto.CreateVMRequest) {
@@ -1692,8 +1695,9 @@ func TestStopVM_Isolated(t *testing.T) {
 			withStopVM: true,
 
 			createVMRequest: proto.CreateVMRequest{
+				KernelArgs: kernelArgs + " failure=slow-reboot",
 				RootDrive: &proto.FirecrackerRootDrive{
-					HostPath: "/var/lib/firecracker-containerd/runtime/rootfs-slow-reboot.img",
+					HostPath: "/var/lib/firecracker-containerd/runtime/rootfs-debug.img",
 				},
 			},
 			stopFunc: func(ctx context.Context, fcClient fccontrol.FirecrackerService, req proto.CreateVMRequest) {
@@ -2137,6 +2141,8 @@ func TestCreateVM_Isolated(t *testing.T) {
 	fcClient, err := newFCControlClient(containerdSockPath)
 	require.NoError(t, err, "failed to create ttrpc client")
 
+	kernelArgs := integtest.DefaultRuntimeConfig.KernelArgs
+
 	type subtest struct {
 		name                    string
 		request                 proto.CreateVMRequest
@@ -2155,11 +2161,12 @@ func TestCreateVM_Isolated(t *testing.T) {
 			stopVM: true,
 		},
 		{
-			name: "Error Case",
+			name: "No Agent",
 			request: proto.CreateVMRequest{
 				TimeoutSeconds: 5,
+				KernelArgs:     kernelArgs + " failure=no-agent",
 				RootDrive: &proto.FirecrackerRootDrive{
-					HostPath: "/var/lib/firecracker-containerd/runtime/rootfs-no-agent.img",
+					HostPath: "/var/lib/firecracker-containerd/runtime/rootfs-debug.img",
 				},
 			},
 			validate: func(t *testing.T, err error) {
@@ -2175,8 +2182,9 @@ func TestCreateVM_Isolated(t *testing.T) {
 		{
 			name: "Slow Root FS",
 			request: proto.CreateVMRequest{
+				KernelArgs: kernelArgs + " failure=slow-boot",
 				RootDrive: &proto.FirecrackerRootDrive{
-					HostPath: "/var/lib/firecracker-containerd/runtime/rootfs-slow-boot.img",
+					HostPath: "/var/lib/firecracker-containerd/runtime/rootfs-debug.img",
 				},
 			},
 			validate: func(t *testing.T, err error) {
@@ -2189,8 +2197,9 @@ func TestCreateVM_Isolated(t *testing.T) {
 		{
 			name: "Slow Root FS and Longer Timeout",
 			request: proto.CreateVMRequest{
+				KernelArgs: kernelArgs + " failure=slow-boot",
 				RootDrive: &proto.FirecrackerRootDrive{
-					HostPath: "/var/lib/firecracker-containerd/runtime/rootfs-slow-boot.img",
+					HostPath: "/var/lib/firecracker-containerd/runtime/rootfs-debug.img",
 				},
 				TimeoutSeconds: 60,
 			},

tools/image-builder/Makefile

@@ -55,7 +55,7 @@ define run_debootstrap =
 	touch $@
 endef
 
-all: rootfs.img rootfs-slow-boot.img rootfs-slow-reboot.img rootfs-no-agent.img
+all: rootfs.img rootfs-debug.img
 stargz: rootfs-stargz.img
 
 $(WORKDIR):
@@ -89,25 +89,12 @@ endif
 rootfs.img: files_common_stamp files_debootstrap_stamp files_ephemeral_stamp
 	mksquashfs "$(WORKDIR)" rootfs.img -noappend
 
-# Intentionally break the rootfs to simulate the case where CreateVM is taking longer than the minimal timeout
-rootfs-slow-boot.img: files_common_stamp files_debootstrap_stamp files_ephemeral_stamp
+rootfs-debug.img: files_common_stamp files_debootstrap_stamp files_ephemeral_stamp
 	rm -fr tmp/$@
 	cp -a "$(WORKDIR)" tmp/$@
-	echo 'ExecStartPre=/bin/sleep 30' >> tmp/$@/etc/systemd/system/firecracker-agent.service
-	mksquashfs tmp/$@ $@ -noappend
-
-# Intentionally break the rootfs to simulate the case where StopVM is taking longer than the minimal timeout
-rootfs-slow-reboot.img: files_common_stamp files_debootstrap_stamp files_ephemeral_stamp
-	rm -fr tmp/$@
-	cp -a "$(WORKDIR)" tmp/$@
-	echo 'ExecStop=/bin/sleep 60' >> tmp/$@/etc/systemd/system/firecracker-agent.service
-	mksquashfs tmp/$@ $@ -noappend
-
-rootfs-no-agent.img: files_common_stamp files_debootstrap_stamp files_ephemeral_stamp
-	rm -fr tmp/$@
-	cp -a "$(WORKDIR)" tmp/$@
-	rm tmp/$@/etc/systemd/system/firecracker-agent.service
-	rm tmp/$@/usr/local/bin/agent
+	mv tmp/$@/usr/local/bin/agent tmp/$@/usr/local/bin/agent.real
+	cp agent.sh tmp/$@/usr/local/bin/agent
+	ls -l tmp/$@/usr/local/bin
 	mksquashfs tmp/$@ $@ -noappend
 
 rootfs-stargz.img: WORKDIR    := $(addsuffix -stargz, $(WORKDIR))
@@ -141,7 +128,7 @@ builder_stamp:
 clean:
 	-rm -f *stamp
 	if [ $(UID) -eq 0 ]; then \
-	  rm -f rootfs.img rootfs-slow-boot.img rootfs-slow-reboot.img rootfs-stargz.img;\
+	  rm -f rootfs.img rootfs-debug.img rootfs-stargz.img;\
 	else \
 	  $(MAKE) clean-in-docker ;\
 	fi

tools/image-builder/agent.sh

@@ -0,0 +1,33 @@
+#! /bin/bash
+# This script adds multiple failure modes to firecracker-containerd's in-VM agent.
+set -euo pipefail
+
+agent=/usr/local/bin/agent.real
+
+if [[ "$(cat /proc/cmdline)" =~ failure=([a-z-]+) ]];
+then
+    failure="${BASH_REMATCH[1]}"
+else
+    failure='none'
+fi
+
+case "$failure" in 
+slow-boot)
+    sleep 30
+    $agent
+    ;;
+slow-reboot)
+    $agent
+    sleep 30
+    ;;
+no-agent)
+    exit 1
+    ;;
+none)
+    $agent
+    ;;
+*)
+    echo "unknown failure mode: $failure"
+    exit 1
+esac
+