Skip to content

Commit c5d57b4

Browse files
artemvminartemvmin
committed
Add E2E test for underspecified, missing volume during unpublish
1 parent 72f2697 commit c5d57b4

File tree

4 files changed

+204
-50
lines changed

4 files changed

+204
-50
lines changed

‎test/e2e/tests/setup_e2e_test.go‎

Lines changed: 37 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,6 @@ func init() {
5656
}
5757

5858
func TestE2E(t *testing.T) {
59-
6059
flag.Parse()
6160
RegisterFailHandler(Fail)
6261
RunSpecs(t, "Google Compute Engine Persistent Disk Container Storage Interface Driver Tests")
@@ -91,53 +90,23 @@ var _ = BeforeSuite(func() {
9190
Expect(*project).ToNot(BeEmpty(), "Project should not be empty")
9291
Expect(*serviceAccount).ToNot(BeEmpty(), "Service account should not be empty")
9392

94-
klog.Infof("Running in project %v with service account %v\n\n", *project, *serviceAccount)
93+
klog.Infof("Running in project %v with service account %v", *project, *serviceAccount)
9594

9695
for _, zone := range zones {
9796
go func(curZone string) {
9897
defer GinkgoRecover()
99-
nodeID := fmt.Sprintf("gce-pd-csi-e2e-%s", curZone)
100-
klog.Infof("Setting up node %s\n", nodeID)
101-
102-
i, err := remote.SetupInstance(*project, *architecture, curZone, nodeID, *machineType, *serviceAccount, *imageURL, computeService)
103-
if err != nil {
104-
klog.Fatalf("Failed to setup instance %v: %w", nodeID, err)
105-
}
106-
107-
err = testutils.MkdirAll(i, "/lib/udev_containerized")
108-
if err != nil {
109-
klog.Fatalf("Could not make scsi_id containerized directory: %w", err)
110-
}
111-
112-
err = testutils.CopyFile(i, "/lib/udev/scsi_id", "/lib/udev_containerized/scsi_id")
113-
if err != nil {
114-
klog.Fatalf("could not copy scsi_id to containerized directory: %w", err)
115-
}
116-
117-
err = testutils.CopyFile(i, "/lib/udev/google_nvme_id", "/lib/udev_containerized/google_nvme_id")
118-
if err != nil {
119-
klog.Fatalf("could not copy google_nvme_id to containerized directory: %w", err)
120-
}
121-
122-
klog.Infof("Creating new driver and client for node %s\n", i.GetName())
123-
// Create new driver and client
124-
testContext, err := testutils.GCEClientAndDriverSetup(i, "")
125-
if err != nil {
126-
klog.Fatalf("Failed to set up Test Context for instance %v: %w", i.GetName(), err)
127-
}
128-
tcc <- testContext
98+
tcc <- NewTestContext(curZone)
12999
}(zone)
130100
}
131101

132102
for i := 0; i < len(zones); i++ {
133103
tc := <-tcc
134-
klog.Infof("Test Context for node %s set up\n", tc.Instance.GetName())
135104
testContexts = append(testContexts, tc)
105+
klog.Infof("Added TestContext for node %s", tc.Instance.GetName())
136106
}
137107
})
138108

139109
var _ = AfterSuite(func() {
140-
141110
for _, tc := range testContexts {
142111
err := remote.TeardownDriverAndClient(tc)
143112
Expect(err).To(BeNil(), "Teardown Driver and Client failed with error")
@@ -147,6 +116,40 @@ var _ = AfterSuite(func() {
147116
}
148117
})
149118

119+
func NewTestContext(zone string) *remote.TestContext {
120+
nodeID := fmt.Sprintf("gce-pd-csi-e2e-%s", zone)
121+
klog.Infof("Setting up node %s", nodeID)
122+
123+
i, err := remote.SetupInstance(*project, *architecture, zone, nodeID, *machineType, *serviceAccount, *imageURL, computeService)
124+
if err != nil {
125+
klog.Fatalf("Failed to setup instance %v: %w", nodeID, err)
126+
}
127+
128+
err = testutils.MkdirAll(i, "/lib/udev_containerized")
129+
if err != nil {
130+
klog.Fatalf("Failed to make scsi_id containerized directory: %w", err)
131+
}
132+
133+
err = testutils.CopyFile(i, "/lib/udev/scsi_id", "/lib/udev_containerized/scsi_id")
134+
if err != nil {
135+
klog.Fatalf("Failed to copy scsi_id to containerized directory: %w", err)
136+
}
137+
138+
err = testutils.CopyFile(i, "/lib/udev/google_nvme_id", "/lib/udev_containerized/google_nvme_id")
139+
if err != nil {
140+
klog.Fatalf("Failed to copy google_nvme_id to containerized directory: %w", err)
141+
}
142+
143+
klog.Infof("Creating new driver and client for node %s", i.GetName())
144+
tc, err := testutils.GCEClientAndDriverSetup(i, "")
145+
if err != nil {
146+
klog.Fatalf("Failed to set up TestContext for instance %v: %w", i.GetName(), err)
147+
}
148+
149+
klog.Infof("Finished creating TestContext for node %s", tc.Instance.GetName())
150+
return tc
151+
}
152+
150153
func getRandomTestContext() *remote.TestContext {
151154
Expect(testContexts).ToNot(BeEmpty())
152155
rn := rand.Intn(len(testContexts))

‎test/e2e/tests/single_zone_e2e_test.go‎

Lines changed: 76 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ import (
3535
. "github.com/onsi/ginkgo"
3636
. "github.com/onsi/gomega"
3737

38+
compute "google.golang.org/api/compute/v1"
3839
"google.golang.org/api/iterator"
3940
kmspb "google.golang.org/genproto/googleapis/cloud/kms/v1"
4041
fieldmask "google.golang.org/genproto/protobuf/field_mask"
@@ -257,7 +258,6 @@ var _ = Describe("GCE PD CSI Driver", func() {
257258
_, err = computeService.Disks.Get(p, zone, volName).Do()
258259
Expect(err).To(BeNil(), "Could not find disk in correct zone")
259260
}
260-
261261
})
262262

263263
It("Should complete entire disk lifecycle with underspecified volume ID", func() {
@@ -284,7 +284,40 @@ var _ = Describe("GCE PD CSI Driver", func() {
284284
// Attach Disk
285285
err := testAttachWriteReadDetach(underSpecifiedID, volName, instance, client, false /* readOnly */)
286286
Expect(err).To(BeNil(), "Failed to go through volume lifecycle")
287+
})
288+
289+
It("Should complete publish/unpublish lifecycle with underspecified volume ID and missing volume", func() {
290+
testContext := getRandomTestContext()
291+
292+
p, z, _ := testContext.Instance.GetIdentity()
293+
client := testContext.Client
294+
instance := testContext.Instance
295+
296+
// Create Disk
297+
volName, _ := createAndValidateUniqueZonalDisk(client, p, z, standardDiskType)
298+
underSpecifiedID := common.GenerateUnderspecifiedVolumeID(volName, true /* isZonal */)
299+
300+
defer func() {
301+
// Detach Disk
302+
err := instance.DetachDisk(volName)
303+
Expect(err).To(BeNil(), "DetachDisk failed")
287304

305+
// Delete Disk
306+
err = client.DeleteVolume(underSpecifiedID)
307+
Expect(err).To(BeNil(), "DeleteVolume failed")
308+
309+
// Validate Disk Deleted
310+
_, err = computeService.Disks.Get(p, z, volName).Do()
311+
Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected disk to not be found")
312+
313+
// Unpublish Disk
314+
err = client.ControllerUnpublishVolume(underSpecifiedID, instance.GetNodeID())
315+
Expect(err).To(BeNil(), "ControllerUnpublishVolume failed")
316+
}()
317+
318+
// Attach Disk
319+
err := client.ControllerPublishVolume(underSpecifiedID, instance.GetNodeID())
320+
Expect(err).To(BeNil(), "ControllerPublishVolume failed")
288321
})
289322

290323
It("Should successfully create RePD in two zones in the drivers region when none are specified", func() {
@@ -1122,11 +1155,11 @@ func equalWithinEpsilon(a, b, epsiolon int64) bool {
11221155
return b-a < epsiolon
11231156
}
11241157

1125-
func createAndValidateUniqueZonalDisk(client *remote.CsiClient, project, zone string, diskType string) (volName, volID string) {
1158+
func createAndValidateUniqueZonalDisk(client *remote.CsiClient, project, zone string, diskType string) (string, string) {
11261159
// Create Disk
1127-
var err error
1128-
volName = testNamePrefix + string(uuid.NewUUID())
1129-
volID, err = client.CreateVolume(volName, nil, defaultSizeGb,
1160+
disk := typeToDisk[diskType]
1161+
volName := testNamePrefix + string(uuid.NewUUID())
1162+
volID, err := client.CreateVolume(volName, disk.params, defaultSizeGb,
11301163
&csi.TopologyRequirement{
11311164
Requisite: []*csi.Topology{
11321165
{
@@ -1139,11 +1172,12 @@ func createAndValidateUniqueZonalDisk(client *remote.CsiClient, project, zone st
11391172
// Validate Disk Created
11401173
cloudDisk, err := computeService.Disks.Get(project, zone, volName).Do()
11411174
Expect(err).To(BeNil(), "Could not get disk from cloud directly")
1142-
Expect(cloudDisk.Type).To(ContainSubstring(diskType))
11431175
Expect(cloudDisk.Status).To(Equal(readyState))
11441176
Expect(cloudDisk.SizeGb).To(Equal(defaultSizeGb))
11451177
Expect(cloudDisk.Name).To(Equal(volName))
1146-
return
1178+
disk.validate(cloudDisk)
1179+
1180+
return volName, volID
11471181
}
11481182

11491183
func deleteVolumeOrError(client *remote.CsiClient, volID string) {
@@ -1160,8 +1194,9 @@ func deleteVolumeOrError(client *remote.CsiClient, volID string) {
11601194

11611195
func createAndValidateUniqueZonalMultiWriterDisk(client *remote.CsiClient, project, zone string, diskType string) (string, string) {
11621196
// Create Disk
1197+
disk := typeToDisk[diskType]
11631198
volName := testNamePrefix + string(uuid.NewUUID())
1164-
volID, err := client.CreateVolumeWithCaps(volName, nil, defaultMwSizeGb,
1199+
volID, err := client.CreateVolumeWithCaps(volName, disk.params, defaultMwSizeGb,
11651200
&csi.TopologyRequirement{
11661201
Requisite: []*csi.Topology{
11671202
{
@@ -1182,13 +1217,16 @@ func createAndValidateUniqueZonalMultiWriterDisk(client *remote.CsiClient, proje
11821217
Expect(err).To(BeNil(), "CreateVolume failed with error: %v", err)
11831218

11841219
// Validate Disk Created
1185-
cloudDisk, err := computeAlphaService.Disks.Get(project, zone, volName).Do()
1186-
Expect(err).To(BeNil(), "Could not get disk from cloud directly")
1187-
Expect(cloudDisk.Type).To(ContainSubstring(diskType))
1220+
cloudDisk, err := computeService.Disks.Get(project, zone, volName).Do()
1221+
Expect(err).To(BeNil(), "Failed to get cloud disk")
11881222
Expect(cloudDisk.Status).To(Equal(readyState))
11891223
Expect(cloudDisk.SizeGb).To(Equal(defaultMwSizeGb))
11901224
Expect(cloudDisk.Name).To(Equal(volName))
1191-
Expect(cloudDisk.MultiWriter).To(Equal(true))
1225+
disk.validate(cloudDisk)
1226+
1227+
alphaDisk, err := computeAlphaService.Disks.Get(project, zone, volName).Do()
1228+
Expect(err).To(BeNil(), "Failed to get cloud disk using alpha API")
1229+
Expect(alphaDisk.MultiWriter).To(Equal(true))
11921230

11931231
return volName, volID
11941232
}
@@ -1247,3 +1285,29 @@ func setupKeyRing(ctx context.Context, parentName string, keyRingId string) (*km
12471285
}
12481286
return key, keyVersions
12491287
}
1288+
1289+
type disk struct {
1290+
params map[string]string
1291+
validate func(disk *compute.Disk)
1292+
}
1293+
1294+
var typeToDisk = map[string]*disk{
1295+
standardDiskType: {
1296+
params: map[string]string{
1297+
common.ParameterKeyType: standardDiskType,
1298+
},
1299+
validate: func(disk *compute.Disk) {
1300+
Expect(disk.Type).To(ContainSubstring(standardDiskType))
1301+
},
1302+
},
1303+
extremeDiskType: {
1304+
params: map[string]string{
1305+
common.ParameterKeyType: extremeDiskType,
1306+
common.ParameterKeyProvisionedIOPSOnCreate: provisionedIOPSOnCreate,
1307+
},
1308+
validate: func(disk *compute.Disk) {
1309+
Expect(disk.Type).To(ContainSubstring(extremeDiskType))
1310+
Expect(disk.ProvisionedIops).To(Equal(provisionedIOPSOnCreateInt))
1311+
},
1312+
},
1313+
}

‎test/e2e/tests/single_zone_pd_extreme_e2e_test.go‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -272,4 +272,64 @@ var _ = Describe("GCE PD CSI Driver pd-extreme", func() {
272272
Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected disk to not be found")
273273
}()
274274
})
275+
276+
It("Should complete entire disk lifecycle with underspecified volume ID", func() {
277+
testContext := getRandomTestContext()
278+
279+
p, z, _ := testContext.Instance.GetIdentity()
280+
client := testContext.Client
281+
instance := testContext.Instance
282+
283+
volName, _ := createAndValidateUniqueZonalDisk(client, p, z, extremeDiskType)
284+
285+
underSpecifiedID := common.GenerateUnderspecifiedVolumeID(volName, true /* isZonal */)
286+
287+
defer func() {
288+
// Delete Disk
289+
err := client.DeleteVolume(underSpecifiedID)
290+
Expect(err).To(BeNil(), "DeleteVolume failed")
291+
292+
// Validate Disk Deleted
293+
_, err = computeService.Disks.Get(p, z, volName).Do()
294+
Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected disk to not be found")
295+
}()
296+
297+
// Attach Disk
298+
err := testAttachWriteReadDetach(underSpecifiedID, volName, instance, client, false /* readOnly */)
299+
Expect(err).To(BeNil(), "Failed to go through volume lifecycle")
300+
})
301+
302+
It("Should complete entire publish/unpublish lifecycle with underspecified volume ID and missing volume", func() {
303+
testContext := getRandomTestContext()
304+
305+
p, z, _ := testContext.Instance.GetIdentity()
306+
client := testContext.Client
307+
instance := testContext.Instance
308+
309+
// Create Disk
310+
volName, _ := createAndValidateUniqueZonalDisk(client, p, z, extremeDiskType)
311+
underSpecifiedID := common.GenerateUnderspecifiedVolumeID(volName, true /* isZonal */)
312+
313+
defer func() {
314+
// Detach Disk
315+
err := instance.DetachDisk(volName)
316+
Expect(err).To(BeNil(), "DetachDisk failed")
317+
318+
// Delete Disk
319+
err = client.DeleteVolume(underSpecifiedID)
320+
Expect(err).To(BeNil(), "DeleteVolume failed")
321+
322+
// Validate Disk Deleted
323+
_, err = computeService.Disks.Get(p, z, volName).Do()
324+
Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected disk to not be found")
325+
326+
// Unpublish Disk
327+
err = client.ControllerUnpublishVolume(underSpecifiedID, instance.GetNodeID())
328+
Expect(err).To(BeNil(), "ControllerUnpublishVolume failed")
329+
}()
330+
331+
// Attach Disk
332+
err := client.ControllerPublishVolume(underSpecifiedID, instance.GetNodeID())
333+
Expect(err).To(BeNil(), "ControllerPublishVolume failed")
334+
})
275335
})

‎test/remote/instance.go‎

Lines changed: 31 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -143,9 +143,9 @@ func (i *InstanceInfo) CreateOrGetInstance(imageURL, serviceAccount string) erro
143143
return err
144144
}
145145

146-
then := time.Now()
146+
start := time.Now()
147147
err := wait.Poll(15*time.Second, 5*time.Minute, func() (bool, error) {
148-
klog.V(2).Infof("Waiting for instance to be deleted. %v elapsed", time.Since(then))
148+
klog.V(2).Infof("Waiting for instance to be deleted. %v elapsed", time.Since(start))
149149
if curInst, _ = i.computeService.Instances.Get(i.project, i.zone, i.name).Do(); curInst != nil {
150150
return false, nil
151151
}
@@ -173,9 +173,9 @@ func (i *InstanceInfo) CreateOrGetInstance(imageURL, serviceAccount string) erro
173173
klog.V(4).Infof("Compute service GOT instance %v, skipping instance creation", newInst.Name)
174174
}
175175

176-
then := time.Now()
176+
start := time.Now()
177177
err = wait.Poll(15*time.Second, 5*time.Minute, func() (bool, error) {
178-
klog.V(2).Infof("Waiting for instance %v to come up. %v elapsed", i.name, time.Since(then))
178+
klog.V(2).Infof("Waiting for instance %v to come up. %v elapsed", i.name, time.Since(start))
179179

180180
instance, err = i.computeService.Instances.Get(i.project, i.zone, i.name).Do()
181181
if err != nil {
@@ -223,6 +223,33 @@ func (i *InstanceInfo) DeleteInstance() {
223223
}
224224
}
225225

226+
func (i *InstanceInfo) DetachDisk(diskName string) error {
227+
klog.V(4).Infof("Detaching disk %q", diskName)
228+
op, err := i.computeService.Instances.DetachDisk(i.project, i.zone, i.name, diskName).Do()
229+
if err != nil {
230+
if isGCEError(err, "notFound") {
231+
return nil
232+
}
233+
klog.Errorf("Error deleting disk %q: %w", diskName, err)
234+
}
235+
236+
start := time.Now()
237+
if err := wait.Poll(5*time.Second, 1*time.Minute, func() (bool, error) {
238+
klog.V(2).Infof("Waiting for disk %q to be detached from instance %q. %v elapsed", diskName, i.name, time.Since(start))
239+
240+
op, err = i.computeService.ZoneOperations.Get(i.project, i.zone, op.Name).Do()
241+
if err != nil {
242+
return true, fmt.Errorf("Failed to get operation %q, err: %v", op.Name, err)
243+
}
244+
return op.Status == "DONE", nil
245+
}); err != nil {
246+
return err
247+
}
248+
249+
klog.V(4).Infof("Disk %q has been successfully detached from instance %q\n%v", diskName, i.name, op.Error)
250+
return nil
251+
}
252+
226253
func getexternalIP(instance *compute.Instance) string {
227254
for i := range instance.NetworkInterfaces {
228255
ni := instance.NetworkInterfaces[i]

0 commit comments

Comments
 (0)