Merge pull request #627 from austinvazquez/integ-test

refactor: integration test common code into internal module

Author: austinvazquez

internal/integtest/config.go

@@ -0,0 +1,62 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"). You may
+// not use this file except in compliance with the License. A copy of the
+// License is located at
+//
+//	http://aws.amazon.com/apache2.0/
+//
+// or in the "license" file accompanying this file. This file is distributed
+// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+// express or implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package integtest
+
+import (
+	"encoding/json"
+	"os"
+
+	"github.com/firecracker-microvm/firecracker-containerd/config"
+)
+
+const runtimeConfigPath = "/etc/containerd/firecracker-runtime.json"
+
+// DefaultRuntimeConfig represents a simple firecracker-containerd configuration.
+var DefaultRuntimeConfig = config.Config{
+	FirecrackerBinaryPath: "/usr/local/bin/firecracker",
+	KernelImagePath:       "/var/lib/firecracker-containerd/runtime/default-vmlinux.bin",
+	KernelArgs:            "ro console=ttyS0 noapic reboot=k panic=1 pci=off nomodules systemd.unified_cgroup_hierarchy=0 systemd.journald.forward_to_console systemd.log_color=false systemd.unit=firecracker.target init=/sbin/overlay-init",
+	RootDrive:             "/var/lib/firecracker-containerd/runtime/default-rootfs.img",
+	LogLevels:             []string{"debug"},
+	ShimBaseDir:           ShimBaseDir(),
+	JailerConfig: config.JailerConfig{
+		RuncBinaryPath: "/usr/local/bin/runc",
+		RuncConfigPath: "/etc/containerd/firecracker-runc-config.json",
+	},
+}
+
+func writeRuntimeConfig(options ...func(*config.Config)) error {
+	config := DefaultRuntimeConfig
+	for _, option := range options {
+		option(&config)
+	}
+
+	file, err := os.OpenFile(runtimeConfigPath, os.O_CREATE|os.O_WRONLY, 0600)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	bytes, err := json.Marshal(config)
+	if err != nil {
+		return err
+	}
+
+	_, err = file.Write(bytes)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

internal/integtest/prepare.go

@@ -0,0 +1,34 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"). You may
+// not use this file except in compliance with the License. A copy of the
+// License is located at
+//
+//	http://aws.amazon.com/apache2.0/
+//
+// or in the "license" file accompanying this file. This file is distributed
+// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+// express or implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package integtest
+
+import (
+	"testing"
+
+	"github.com/firecracker-microvm/firecracker-containerd/config"
+	"github.com/firecracker-microvm/firecracker-containerd/internal"
+)
+
+// Prepare is a common integration test setup function which ensures
+// isolation and prepares the runtime configuration for firecracker-containerd
+func Prepare(t testing.TB, options ...func(*config.Config)) {
+	t.Helper()
+
+	internal.RequiresIsolation(t)
+
+	err := writeRuntimeConfig(options...)
+	if err != nil {
+		t.Error(err)
+	}
+}

internal/integtest/shim.go

@@ -0,0 +1,29 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"). You may
+// not use this file except in compliance with the License. A copy of the
+// License is located at
+//
+//	http://aws.amazon.com/apache2.0/
+//
+// or in the "license" file accompanying this file. This file is distributed
+// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+// express or implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package integtest
+
+import "os"
+
+const shimBaseDirEnvVar = "SHIM_BASE_DIR"
+const defaultShimBaseDir = "/srv/firecracker_containerd_tests"
+
+// ShimBaseDir checks the "SHIM_BASE_DIR" environment variable and returns its value
+// if it exists, else returns the default value
+func ShimBaseDir() string {
+	if v := os.Getenv(shimBaseDirEnvVar); v != "" {
+		return v
+	}
+
+	return defaultShimBaseDir
+}

runtime/benchmark_test.go

@@ -20,6 +20,7 @@ import (
 
 	"github.com/containerd/containerd"
 	"github.com/containerd/containerd/namespaces"
+	"github.com/firecracker-microvm/firecracker-containerd/internal/integtest"
 	"github.com/firecracker-microvm/firecracker-containerd/proto"
 	fccontrol "github.com/firecracker-microvm/firecracker-containerd/proto/service/fccontrol/ttrpc"
 	"github.com/gofrs/uuid"
@@ -94,13 +95,13 @@ func benchmarkCreateAndStopVM(t *testing.T, vcpuCount uint32, kernelArgs string)
 }
 
 func TestPerf_CreateVM(t *testing.T) {
-	prepareIntegTest(t)
+	integtest.Prepare(t)
 
 	t.Run("vcpu=1", func(subtest *testing.T) {
-		benchmarkCreateAndStopVM(subtest, 1, defaultRuntimeConfig.KernelArgs)
+		benchmarkCreateAndStopVM(subtest, 1, integtest.DefaultRuntimeConfig.KernelArgs)
 	})
 	t.Run("vcpu=5", func(subtest *testing.T) {
-		benchmarkCreateAndStopVM(subtest, 1, defaultRuntimeConfig.KernelArgs)
+		benchmarkCreateAndStopVM(subtest, 1, integtest.DefaultRuntimeConfig.KernelArgs)
 	})
 	t.Run("firecracker-demo", func(subtest *testing.T) {
 		benchmarkCreateAndStopVM(

runtime/cni_integ_test.go

@@ -34,12 +34,13 @@ import (
 
 	"github.com/firecracker-microvm/firecracker-containerd/config"
 	"github.com/firecracker-microvm/firecracker-containerd/internal"
+	"github.com/firecracker-microvm/firecracker-containerd/internal/integtest"
 	"github.com/firecracker-microvm/firecracker-containerd/proto"
 	"github.com/firecracker-microvm/firecracker-containerd/runtime/firecrackeroci"
 )
 
 func TestCNISupport_Isolated(t *testing.T) {
-	prepareIntegTest(t)
+	integtest.Prepare(t)
 	testTimeout := 120 * time.Second
 	ctx, cancel := context.WithTimeout(namespaces.WithNamespace(context.Background(), defaultNamespace), testTimeout)
 	defer cancel()
@@ -147,7 +148,7 @@ func TestCNISupport_Isolated(t *testing.T) {
 }
 
 func TestAutomaticCNISupport_Isolated(t *testing.T) {
-	prepareIntegTest(t, withDefaultNetwork())
+	integtest.Prepare(t, withDefaultNetwork())
 
 	testTimeout := 120 * time.Second
 	ctx, cancel := context.WithTimeout(namespaces.WithNamespace(context.Background(), defaultNamespace), testTimeout)
@@ -213,7 +214,7 @@ func TestAutomaticCNISupport_Isolated(t *testing.T) {
 }
 
 func TestCNIPlugin_Performance(t *testing.T) {
-	prepareIntegTest(t)
+	integtest.Prepare(t)
 
 	numVMs := perfTestVMCount(t)
 	runtimeDuration := perfTestRuntime(t)

runtime/integ_test.go

@@ -15,95 +15,31 @@ package main
 import (
 	"bytes"
 	"context"
-	"encoding/json"
-	"os"
 	"strings"
-	"testing"
 
 	"github.com/containerd/containerd"
 	"github.com/containerd/containerd/cio"
 	"github.com/pkg/errors"
 
-	"github.com/firecracker-microvm/firecracker-containerd/config"
 	"github.com/firecracker-microvm/firecracker-containerd/firecracker-control/client"
 	"github.com/firecracker-microvm/firecracker-containerd/internal"
+	"github.com/firecracker-microvm/firecracker-containerd/internal/integtest"
 )
 
-const runtimeConfigPath = "/etc/containerd/firecracker-runtime.json"
-const shimBaseDirEnvVar = "SHIM_BASE_DIR"
-const defaultShimBaseDir = "/srv/firecracker_containerd_tests"
-
-var defaultRuntimeConfig = config.Config{
-	FirecrackerBinaryPath: "/usr/local/bin/firecracker",
-	KernelImagePath:       "/var/lib/firecracker-containerd/runtime/default-vmlinux.bin",
-	KernelArgs:            "ro console=ttyS0 noapic reboot=k panic=1 pci=off nomodules systemd.unified_cgroup_hierarchy=0 systemd.journald.forward_to_console systemd.log_color=false systemd.unit=firecracker.target init=/sbin/overlay-init",
-	RootDrive:             "/var/lib/firecracker-containerd/runtime/default-rootfs.img",
-	LogLevels:             []string{"debug"},
-	ShimBaseDir:           shimBaseDir(),
-	JailerConfig: config.JailerConfig{
-		RuncBinaryPath: "/usr/local/bin/runc",
-		RuncConfigPath: "/etc/containerd/firecracker-runc-config.json",
-	},
-}
-
 func init() {
 	flag, err := internal.SupportCPUTemplate()
 	if err != nil {
 		panic(err)
 	}
 
 	if flag {
-		defaultRuntimeConfig.CPUTemplate = "T2"
+		integtest.DefaultRuntimeConfig.CPUTemplate = "T2"
 	}
 }
 
-func shimBaseDir() string {
-	if v := os.Getenv(shimBaseDirEnvVar); v != "" {
-		return v
-	}
-
-	return defaultShimBaseDir
-}
-
 // devmapper is the only snapshotter we can use with Firecracker
 const defaultSnapshotterName = "devmapper"
 
-func prepareIntegTest(t testing.TB, options ...func(*config.Config)) {
-	t.Helper()
-
-	internal.RequiresIsolation(t)
-
-	err := writeRuntimeConfig(options...)
-	if err != nil {
-		t.Error(err)
-	}
-}
-
-func writeRuntimeConfig(options ...func(*config.Config)) error {
-	config := defaultRuntimeConfig
-	for _, option := range options {
-		option(&config)
-	}
-
-	file, err := os.OpenFile(runtimeConfigPath, os.O_CREATE|os.O_WRONLY, 0600)
-	if err != nil {
-		return err
-	}
-	defer file.Close()
-
-	bytes, err := json.Marshal(config)
-	if err != nil {
-		return err
-	}
-
-	_, err = file.Write(bytes)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
 var testNameToVMIDReplacer = strings.NewReplacer("/", "-", "_", "-")
 
 func testNameToVMID(s string) string {

runtime/jailer_integ_test.go

@@ -30,6 +30,7 @@ import (
 
 	_ "github.com/firecracker-microvm/firecracker-containerd/firecracker-control"
 	"github.com/firecracker-microvm/firecracker-containerd/internal"
+	"github.com/firecracker-microvm/firecracker-containerd/internal/integtest"
 	"github.com/firecracker-microvm/firecracker-containerd/proto"
 	"github.com/firecracker-microvm/firecracker-containerd/runtime/cpuset"
 	"github.com/firecracker-microvm/firecracker-containerd/runtime/firecrackeroci"
@@ -41,7 +42,7 @@ const (
 )
 
 func TestJailer_Isolated(t *testing.T) {
-	prepareIntegTest(t)
+	integtest.Prepare(t)
 	t.Run("Without Jailer", func(t *testing.T) {
 		t.Parallel()
 		testJailer(t, nil)
@@ -64,7 +65,7 @@ func TestJailer_Isolated(t *testing.T) {
 }
 
 func TestAttachBlockDevice_Isolated(t *testing.T) {
-	prepareIntegTest(t)
+	integtest.Prepare(t)
 	t.Run("Without Jailer", func(t *testing.T) {
 		t.Parallel()
 		testAttachBlockDevice(t, nil)
@@ -126,7 +127,7 @@ func testJailer(t *testing.T, jailerConfig *proto.JailerConfig) {
 		dst := f.Name()
 
 		// Copy the root drive before chown, since the file is used by other tests.
-		err = copyFile(defaultRuntimeConfig.RootDrive, dst, 0400)
+		err = copyFile(integtest.DefaultRuntimeConfig.RootDrive, dst, 0400)
 		require.NoErrorf(err, "failed to copy a rootfs as %q", dst)
 
 		err = os.Chown(dst, int(jailerConfig.UID), int(jailerConfig.GID))
@@ -166,7 +167,7 @@ func testJailer(t *testing.T, jailerConfig *proto.JailerConfig) {
 	stdout := startAndWaitTask(ctx, t, c)
 	require.Equal("hello\nadditional drive\n", stdout)
 
-	stat, err := os.Stat(filepath.Join(shimBaseDir(), "default#"+vmID))
+	stat, err := os.Stat(filepath.Join(integtest.ShimBaseDir(), "default#"+vmID))
 	require.NoError(err)
 	assert.True(t, stat.IsDir())
 
@@ -176,17 +177,17 @@ func testJailer(t *testing.T, jailerConfig *proto.JailerConfig) {
 	_, err = fcClient.StopVM(ctx, &proto.StopVMRequest{VMID: vmID})
 	require.NoError(err)
 
-	_, err = os.Stat(filepath.Join(shimBaseDir(), "default#"+vmID))
+	_, err = os.Stat(filepath.Join(integtest.ShimBaseDir(), "default#"+vmID))
 	assert.Error(t, err)
 	assert.True(t, os.IsNotExist(err))
 
-	shimContents, err := ioutil.ReadDir(shimBaseDir())
+	shimContents, err := ioutil.ReadDir(integtest.ShimBaseDir())
 	require.NoError(err)
 	assert.Len(t, shimContents, 0)
 }
 
 func TestJailerCPUSet_Isolated(t *testing.T) {
-	prepareIntegTest(t)
+	integtest.Prepare(t)
 
 	b := cpuset.Builder{}
 	cset := b.AddCPU(0).AddMem(0).Build()
@@ -240,7 +241,7 @@ func testAttachBlockDevice(t *testing.T, jailerConfig *proto.JailerConfig) {
 		dst := f.Name()
 
 		// Copy the root drive before chown, since the file is used by other tests.
-		err = copyFile(defaultRuntimeConfig.RootDrive, dst, 0400)
+		err = copyFile(integtest.DefaultRuntimeConfig.RootDrive, dst, 0400)
 		require.NoErrorf(err, "failed to copy a rootfs as %q", dst)
 
 		err = os.Chown(dst, int(jailerConfig.UID), int(jailerConfig.GID))
@@ -277,7 +278,7 @@ func testAttachBlockDevice(t *testing.T, jailerConfig *proto.JailerConfig) {
 	stdout := startAndWaitTask(ctx, t, c)
 	require.Equal("heyhey\n", stdout)
 
-	stat, err := os.Stat(filepath.Join(shimBaseDir(), "default#"+vmID))
+	stat, err := os.Stat(filepath.Join(integtest.ShimBaseDir(), "default#"+vmID))
 	require.NoError(err)
 	assert.True(t, stat.IsDir())
 
@@ -287,11 +288,11 @@ func testAttachBlockDevice(t *testing.T, jailerConfig *proto.JailerConfig) {
 	_, err = fcClient.StopVM(ctx, &proto.StopVMRequest{VMID: vmID})
 	require.NoError(err)
 
-	_, err = os.Stat(filepath.Join(shimBaseDir(), "default#"+vmID))
+	_, err = os.Stat(filepath.Join(integtest.ShimBaseDir(), "default#"+vmID))
 	assert.Error(t, err)
 	assert.True(t, os.IsNotExist(err))
 
-	shimContents, err := ioutil.ReadDir(shimBaseDir())
+	shimContents, err := ioutil.ReadDir(integtest.ShimBaseDir())
 	require.NoError(err)
 	assert.Len(t, shimContents, 0)
 }

runtime/jailer_test.go

@@ -26,6 +26,7 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/firecracker-microvm/firecracker-containerd/config"
+	"github.com/firecracker-microvm/firecracker-containerd/internal/integtest"
 	"github.com/firecracker-microvm/firecracker-containerd/internal/vm"
 	"github.com/firecracker-microvm/firecracker-containerd/proto"
 )
@@ -99,7 +100,7 @@ func TestJailer_invalidUIDGID(t *testing.T) {
 }
 
 func TestNewJailer_Isolated(t *testing.T) {
-	prepareIntegTest(t)
+	integtest.Prepare(t)
 
 	ociBundlePath := t.TempDir()
 	b, err := exec.Command(