Merge pull request #394 from msau42/regional-tests

Add gke regional cluster support to k8s-integration test

Author: k8s-ci-robot

test/k8s-integration/cluster.go

@@ -10,6 +10,20 @@ import (
 	"k8s.io/klog"
 )
 
+func gkeLocationArgs(gceZone, gceRegion string) (locationArg, locationVal string, err error) {
+	switch {
+	case len(gceZone) > 0:
+		locationArg = "--zone"
+		locationVal = gceZone
+	case len(gceRegion) > 0:
+		locationArg = "--region"
+		locationVal = gceRegion
+	default:
+		return "", "", fmt.Errorf("zone and region unspecified")
+	}
+	return
+}
+
 func clusterDownGCE(k8sDir string) error {
 	cmd := exec.Command(filepath.Join(k8sDir, "hack", "e2e-internal", "e2e-down.sh"))
 	err := runCommand("Bringing Down E2E Cluster on GCE", cmd)
@@ -19,10 +33,15 @@ func clusterDownGCE(k8sDir string) error {
 	return nil
 }
 
-func clusterDownGKE(gceZone string) error {
+func clusterDownGKE(gceZone, gceRegion string) error {
+	locationArg, locationVal, err := gkeLocationArgs(gceZone, gceRegion)
+	if err != nil {
+		return err
+	}
+
 	cmd := exec.Command("gcloud", "container", "clusters", "delete", gkeTestClusterName,
-		"--zone", gceZone, "--quiet")
-	err := runCommand("Bringing Down E2E Cluster on GKE", cmd)
+		locationArg, locationVal, "--quiet")
+	err = runCommand("Bringing Down E2E Cluster on GKE", cmd)
 	if err != nil {
 		return fmt.Errorf("failed to bring down kubernetes e2e cluster on gke: %v", err)
 	}
@@ -77,21 +96,27 @@ func clusterUpGCE(k8sDir, gceZone string, numNodes int) error {
 	return nil
 }
 
-func clusterUpGKE(gceZone string, numNodes int) error {
-	out, err := exec.Command("gcloud", "container", "clusters", "list", "--zone", gceZone,
+func clusterUpGKE(gceZone, gceRegion string, numNodes int) error {
+	locationArg, locationVal, err := gkeLocationArgs(gceZone, gceRegion)
+	if err != nil {
+		return err
+	}
+
+	out, err := exec.Command("gcloud", "container", "clusters", "list", locationArg, locationVal,
 		"--filter", fmt.Sprintf("name=%s", gkeTestClusterName)).CombinedOutput()
+
 	if err != nil {
 		return fmt.Errorf("failed to check for previous test cluster: %v %s", err, out)
 	}
 	if len(out) > 0 {
 		klog.Infof("Detected previous cluster %s. Deleting so a new one can be created...", gkeTestClusterName)
-		err = clusterDownGKE(gceZone)
+		err = clusterDownGKE(gceZone, gceRegion)
 		if err != nil {
 			return err
 		}
 	}
 	cmd := exec.Command("gcloud", "container", "clusters", "create", gkeTestClusterName,
-		"--zone", gceZone, "--cluster-version", *gkeClusterVer, "--num-nodes", strconv.Itoa(numNodes),
+		locationArg, locationVal, "--cluster-version", *gkeClusterVer, "--num-nodes", strconv.Itoa(numNodes),
 		"--quiet", "--machine-type", "n1-standard-2")
 	err = runCommand("Staring E2E Cluster on GKE", cmd)
 	if err != nil {
@@ -157,7 +182,17 @@ func downloadKubernetesSource(pkgDir, k8sIoDir, kubeVersion string) error {
 	return nil
 }
 
-func getGKEKubeTestArgs() ([]string, error) {
+func getGKEKubeTestArgs(gceZone, gceRegion string) ([]string, error) {
+	var locationArg, locationVal string
+	switch {
+	case len(gceZone) > 0:
+		locationArg = "--gcp-zone"
+		locationVal = gceZone
+	case len(gceRegion) > 0:
+		locationArg = "--gcp-region"
+		locationVal = gceRegion
+	}
+
 	var gkeEnv string
 	switch gkeURL := os.Getenv("CLOUDSDK_API_ENDPOINT_OVERRIDES_CONTAINER"); gkeURL {
 	case "https://staging-container.sandbox.googleapis.com/":
@@ -189,7 +224,7 @@ func getGKEKubeTestArgs() ([]string, error) {
 		"--gcp-network=default",
 		fmt.Sprintf("--cluster=%s", gkeTestClusterName),
 		fmt.Sprintf("--gke-environment=%s", gkeEnv),
-		fmt.Sprintf("--gcp-zone=%s", *gceZone),
+		fmt.Sprintf("%s=%s", locationArg, locationVal),
 		fmt.Sprintf("--gcp-project=%s", project[:len(project)-1]),
 	}
 

test/k8s-integration/config/test-config-template.in

@@ -17,6 +17,7 @@ DriverInfo:
     fsGroup: true
     exec: true
     block: true
+    topology: true
     # dataSource: true
     # RWX: true
   SupportedMountOption:

test/k8s-integration/main.go

@@ -34,13 +34,15 @@ var (
 	teardownDriver   = flag.Bool("teardown-driver", true, "teardown the driver after the e2e test")
 	bringupCluster   = flag.Bool("bringup-cluster", true, "build kubernetes and bringup a cluster")
 	gceZone          = flag.String("gce-zone", "", "zone that the gce k8s cluster is created/found in")
+	gceRegion        = flag.String("gce-region", "", "region that gke regional cluster should be created in")
 	kubeVersion      = flag.String("kube-version", "", "version of Kubernetes to download and use for the cluster")
 	testVersion      = flag.String("test-version", "", "version of Kubernetes to download and use for tests")
 	kubeFeatureGates = flag.String("kube-feature-gates", "", "feature gates to set on new kubernetes cluster")
 	localK8sDir      = flag.String("local-k8s-dir", "", "local prebuilt kubernetes/kubernetes directory to use for cluster and test binaries")
-	deploymentStrat  = flag.String("deployment-strategy", "", "choose between deploying on gce or gke")
+	deploymentStrat  = flag.String("deployment-strategy", "gce", "choose between deploying on gce or gke")
 	gkeClusterVer    = flag.String("gke-cluster-version", "", "version of Kubernetes master and node for gke")
 	numNodes         = flag.Int("num-nodes", -1, "the number of nodes in the test cluster")
+
 	// Test infrastructure flags
 	boskosResourceType = flag.String("boskos-resource-type", "gce-project", "name of the boskos resource type to reserve")
 	storageClassFile   = flag.String("storageclass-file", "", "name of storageclass yaml file to use for test relative to test/k8s-integration/config")
@@ -78,7 +80,12 @@ func main() {
 	ensureVariable(saFile, true, "service-account-file is a required flag")
 	ensureVariable(deployOverlayName, true, "deploy-overlay-name is a required flag")
 	ensureVariable(testFocus, true, "test-focus is a required flag")
-	ensureVariable(gceZone, true, "gce-zone is a required flag")
+
+	if len(*gceRegion) != 0 {
+		ensureVariable(gceZone, false, "gce-zone and gce-region cannot both be set")
+	} else {
+		ensureVariable(gceZone, true, "One of gce-zone or gce-region must be set")
+	}
 
 	if *migrationTest {
 		ensureVariable(storageClassFile, false, "storage-class-file and migration-test cannot both be set")
@@ -90,12 +97,6 @@ func main() {
 		ensureVariable(kubeFeatureGates, false, "kube-feature-gates set but not bringing up new cluster")
 	}
 
-	if *bringupCluster || *teardownCluster {
-		ensureVariable(deploymentStrat, true, "Must set the deployment strategy if bringing up or down cluster.")
-	} else {
-		ensureVariable(deploymentStrat, false, "Cannot set the deployment strategy if not bringing up or down cluster.")
-	}
-
 	if *deploymentStrat == "gke" {
 		ensureFlag(migrationTest, false, "Cannot set deployment strategy to 'gke' for migration tests.")
 		ensureVariable(kubeVersion, false, "Cannot set kube-version when using deployment strategy 'gke'. Use gke-cluster-version.")
@@ -104,6 +105,9 @@ func main() {
 		if len(*localK8sDir) == 0 {
 			ensureVariable(testVersion, true, "Must set either test-version or local k8s dir when using deployment strategy 'gke'.")
 		}
+	} else if *deploymentStrat == "gce" {
+		ensureVariable(gceRegion, false, "regional clusters not supported for 'gce' deployment")
+		ensureVariable(gceZone, true, "gce-zone required for 'gce' deployment")
 	}
 
 	if len(*localK8sDir) != 0 {
@@ -229,20 +233,14 @@ func handle() error {
 		testDir = k8sDir
 	}
 
-	var cloudProviderArgs []string
-
 	// Create a cluster either through GKE or GCE
 	if *bringupCluster {
 		var err error = nil
 		switch *deploymentStrat {
 		case "gce":
 			err = clusterUpGCE(k8sDir, *gceZone, *numNodes)
 		case "gke":
-			err = clusterUpGKE(*gceZone, *numNodes)
-			cloudProviderArgs, err = getGKEKubeTestArgs()
-			if err != nil {
-				return fmt.Errorf("failed to build GKE kubetest args: %v", err)
-			}
+			err = clusterUpGKE(*gceZone, *gceRegion, *numNodes)
 		default:
 			err = fmt.Errorf("deployment-strategy must be set to 'gce' or 'gke', but is: %s", *deploymentStrat)
 		}
@@ -261,7 +259,7 @@ func handle() error {
 					klog.Errorf("failed to cluster down: %v", err)
 				}
 			case "gke":
-				err := clusterDownGKE(*gceZone)
+				err := clusterDownGKE(*gceZone, *gceRegion)
 				if err != nil {
 					klog.Errorf("failed to cluster down: %v", err)
 				}
@@ -285,6 +283,15 @@ func handle() error {
 		return fmt.Errorf("failed to install CSI Driver: %v", err)
 	}
 
+	var cloudProviderArgs []string
+	switch *deploymentStrat {
+	case "gke":
+		cloudProviderArgs, err = getGKEKubeTestArgs(*gceZone, *gceRegion)
+		if err != nil {
+			return fmt.Errorf("failed to build GKE kubetest args: %v", err)
+		}
+	}
+
 	// Run the tests using the testDir kubernetes
 	if len(*storageClassFile) != 0 {
 		err = runCSITests(pkgDir, testDir, *testFocus, *storageClassFile, cloudProviderArgs)
@@ -348,6 +355,7 @@ func runTestsWithConfig(testDir, testFocus, testConfigArg string, cloudProviderA
 	kubeTestArgs := []string{
 		"--test",
 		"--ginkgo-parallel",
+		"--check-version-skew=false",
 		fmt.Sprintf("--test_args=%s", testArgs),
 	}