Skip to content

Commit 51c6c35

Browse files
authored
Fix S3 download mkdirp folders instead of writeStream (#482)
## Description I added directory detection with slash ending, separated the directory keys creation to use mkdirp and preserved the current logic for files. ## Motivation the S3 download task failed to create nested objects after creating base directories as 0 byte files. ## Related Issue(s) #480 #468
1 parent af2a783 commit 51c6c35

File tree

2 files changed

+22
-10
lines changed

2 files changed

+22
-10
lines changed
Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
{
2+
"type": "Bug Fix",
3+
"description": "S3Download now correctly handles directories"
4+
}

src/tasks/S3Download/TaskOperations.ts

Lines changed: 18 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -82,20 +82,28 @@ export class TaskOperations {
8282

8383
private async downloadFile(s3Params: S3.GetObjectRequest, dest: string): Promise<void> {
8484
return new Promise<void>((resolve, reject) => {
85-
const dir: string = path.dirname(dest)
86-
if (!fs.existsSync(dir)) {
87-
tl.mkdirP(dir)
85+
const isDir = dest.endsWith('/')
86+
if (isDir) {
87+
this.createDirectory(dest)
88+
} else {
89+
const dir: string = path.dirname(dest)
90+
this.createDirectory(dir)
91+
const file = fs.createWriteStream(dest)
92+
const s3Stream = this.s3Client.getObject(s3Params).createReadStream()
93+
s3Stream.on('error', reject)
94+
file.on('error', reject)
95+
file.on('close', resolve)
96+
s3Stream.pipe(file)
8897
}
89-
90-
const file = fs.createWriteStream(dest)
91-
const s3Stream = this.s3Client.getObject(s3Params).createReadStream()
92-
s3Stream.on('error', reject)
93-
file.on('error', reject)
94-
file.on('close', resolve)
95-
s3Stream.pipe(file)
9698
})
9799
}
98100

101+
private createDirectory(path: string) {
102+
if (!fs.existsSync(path)) {
103+
tl.mkdirP(path)
104+
}
105+
}
106+
99107
private async fetchAllObjectKeys(): Promise<string[]> {
100108
if (this.taskParameters.sourceFolder) {
101109
console.log(

0 commit comments

Comments
 (0)