Skip to content
This repository was archived by the owner on Mar 10, 2020. It is now read-only.

Commit d9569ab

Browse files
Alan Shawachingbrain
andcommitted
refactor: remove Node.js and pull streams (#66)
* refactor: remove Node.js and pull streams Also changes `stat` and `ls` to return CID's not strings from core and removes the `long` option from `ls` in core (is applied by CLI/HTTP API). * fix: cid formatting in files.stat CLI and HTTP API * fix: revert to older dependencies 😢 * chore: eslint-disable-line require-await * fix: stream is objectMode * chore: rebase and fix tests Co-authored-by: Alex Potsides <alex@achingbrain.net>
1 parent c44f925 commit d9569ab

File tree

16 files changed

+178
-270
lines changed

16 files changed

+178
-270
lines changed

package.json

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,6 @@
5454
"ipfs-block-service": "~0.16.0",
5555
"ipfs-repo": "^0.30.1",
5656
"ipld": "~0.25.0",
57-
"it-all": "^1.0.1",
5857
"memdown": "^5.1.0",
5958
"nyc": "^15.0.0",
6059
"sinon": "^8.0.4",
@@ -75,16 +74,16 @@
7574
"ipfs-unixfs": "^0.3.0",
7675
"ipfs-unixfs-exporter": "^0.41.0",
7776
"ipfs-unixfs-importer": "^0.44.0",
78-
"ipfs-utils": "^0.4.2",
77+
"ipfs-utils": "^0.7.0",
7978
"ipld-dag-pb": "^0.18.0",
79+
"it-all": "^1.0.1",
8080
"it-last": "^1.0.1",
81+
"it-to-stream": "^0.1.1",
8182
"it-pipe": "^1.0.1",
8283
"joi-browser": "^13.4.0",
8384
"mortice": "^2.0.0",
8485
"multicodec": "^1.0.0",
85-
"multihashes": "^0.4.14",
86-
"once": "^1.4.0",
87-
"pull-stream": "^3.6.9"
86+
"multihashes": "^0.4.14"
8887
},
8988
"contributors": [
9089
"Alan Shaw <alan.shaw@protocol.ai>",

src/cli/flush.js

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@ module.exports = {
1111

1212
builder: {
1313
'cid-base': {
14-
default: 'base58btc',
1514
describe: 'CID base to use.'
1615
}
1716
},
@@ -28,7 +27,7 @@ module.exports = {
2827
const ipfs = await getIpfs()
2928
let cid = await ipfs.files.flush(path || FILE_SEPARATOR, {})
3029

31-
if (cidBase !== 'base58btc' && cid.version === 0) {
30+
if (cidBase && cidBase !== 'base58btc' && cid.version === 0) {
3231
cid = cid.toV1()
3332
}
3433

src/cli/ls.js

Lines changed: 20 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,6 @@
11
'use strict'
22

3-
const pull = require('pull-stream/pull')
4-
const onEnd = require('pull-stream/sinks/on-end')
5-
const through = require('pull-stream/throughs/through')
3+
const all = require('it-all')
64
const {
75
asBoolean
86
} = require('./utils')
@@ -33,7 +31,6 @@ module.exports = {
3331
describe: 'Sort entries by name'
3432
},
3533
'cid-base': {
36-
default: 'base58btc',
3734
describe: 'CID base to use.'
3835
}
3936
},
@@ -50,53 +47,30 @@ module.exports = {
5047

5148
argv.resolve((async () => {
5249
const ipfs = await getIpfs()
53-
return new Promise((resolve, reject) => {
54-
if (sort) {
55-
ipfs.files.ls(path || FILE_SEPARATOR)
56-
.then(files => {
57-
// https://github.com/ipfs/go-ipfs/issues/5181
58-
if (sort) {
59-
files = files.sort((a, b) => {
60-
return a.name.localeCompare(b.name)
61-
})
62-
}
6350

64-
if (long) {
65-
files.forEach(file => {
66-
print(`${formatMode(file.mode, file.type === 1)}\t${formatMtime(file.mtime)}\t${file.name}\t${file.hash}\t${file.size}`)
67-
})
68-
} else {
69-
files.forEach(link => print(link.name))
70-
}
51+
const printListing = file => {
52+
if (long) {
53+
print(`${formatMode(file.mode, file.type === 1)}\t${formatMtime(file.mtime)}\t${file.name}\t${file.cid.toString(cidBase)}\t${file.size}`)
54+
} else {
55+
print(file.name)
56+
}
57+
}
7158

72-
resolve()
73-
})
74-
.catch(reject)
59+
// https://github.com/ipfs/go-ipfs/issues/5181
60+
if (sort) {
61+
let files = await all(ipfs.files.ls(path || FILE_SEPARATOR))
7562

76-
return
77-
}
63+
files = files.sort((a, b) => {
64+
return a.name.localeCompare(b.name)
65+
})
7866

79-
pull(
80-
ipfs.files.lsPullStream(path, {
81-
long,
82-
cidBase
83-
}),
84-
through(file => {
85-
if (long) {
86-
print(`${formatMode(file.mode, file.type === 1)}\t${formatMtime(file.mtime)}\t${file.name}\t${file.hash}\t${file.size}`)
87-
} else {
88-
print(file.name)
89-
}
90-
}),
91-
onEnd((error) => {
92-
if (error) {
93-
return reject(error)
94-
}
67+
files.forEach(printListing)
68+
return
69+
}
9570

96-
resolve()
97-
})
98-
)
99-
})
71+
for await (const file of ipfs.files.ls(path || FILE_SEPARATOR)) {
72+
printListing(file)
73+
}
10074
})())
10175
}
10276
}

src/cli/read.js

Lines changed: 6 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,5 @@
11
'use strict'
22

3-
const pull = require('pull-stream/pull')
4-
const through = require('pull-stream/throughs/through')
5-
const onEnd = require('pull-stream/sinks/on-end')
6-
73
module.exports = {
84
command: 'read <path>',
95

@@ -34,24 +30,12 @@ module.exports = {
3430
argv.resolve((async () => {
3531
const ipfs = await getIpfs()
3632

37-
return new Promise((resolve, reject) => {
38-
pull(
39-
ipfs.files.readPullStream(path, {
40-
offset,
41-
length
42-
}),
43-
through(buffer => {
44-
print(buffer, false)
45-
}),
46-
onEnd((error) => {
47-
if (error) {
48-
return reject(error)
49-
}
50-
51-
resolve()
52-
})
53-
)
54-
})
33+
for await (const buffer of ipfs.files.read(path, {
34+
offset,
35+
length
36+
})) {
37+
print(buffer, false)
38+
}
5539
})())
5640
}
5741
}

src/cli/stat.js

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,6 @@ Mtime: <mtime>`,
4646
describe: 'Compute the amount of the dag that is local, and if possible the total size'
4747
},
4848
'cid-base': {
49-
default: 'base58btc',
5049
describe: 'CID base to use.'
5150
}
5251
},
@@ -59,7 +58,8 @@ Mtime: <mtime>`,
5958
format,
6059
hash,
6160
size,
62-
withLocal
61+
withLocal,
62+
cidBase
6363
} = argv
6464

6565
argv.resolve((async () => {
@@ -70,15 +70,15 @@ Mtime: <mtime>`,
7070
})
7171
.then((stats) => {
7272
if (hash) {
73-
return print(stats.hash)
73+
return print(stats.cid.toString(cidBase))
7474
}
7575

7676
if (size) {
7777
return print(stats.size)
7878
}
7979

8080
print(format
81-
.replace('<hash>', stats.hash)
81+
.replace('<hash>', stats.cid.toString(cidBase))
8282
.replace('<size>', stats.size)
8383
.replace('<cumulsize>', stats.cumulativeSize)
8484
.replace('<childs>', stats.blocks)

src/core/write.js

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -160,7 +160,7 @@ const write = async (context, source, destination, options) => {
160160
limitAsyncStreamBytes(source, options.length)
161161
)
162162

163-
const content = countBytesStreamed(catAsyncInterators(sources), (bytesWritten) => {
163+
const content = countBytesStreamed(catAsyncIterators(sources), (bytesWritten) => {
164164
if (destination.unixfs && !options.truncate) {
165165
// if we've done reading from the new source and we are not going
166166
// to truncate the file, add the end of the existing file to the output
@@ -254,11 +254,9 @@ const asyncZeroes = (count, chunkSize = MAX_CHUNK_SIZE) => {
254254
return limitAsyncStreamBytes(stream, count)
255255
}
256256

257-
const catAsyncInterators = async function * (sources) {
257+
const catAsyncIterators = async function * (sources) { // eslint-disable-line require-await
258258
for (let i = 0; i < sources.length; i++) {
259-
for await (const buf of sources[i]()) {
260-
yield buf
261-
}
259+
yield * sources[i]()
262260
}
263261
}
264262

src/http/flush.js

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ const mfsFlush = {
2020

2121
let cid = await ipfs.files.flush(arg || FILE_SEPARATOR, {})
2222

23-
if (cidBase !== 'base58btc' && cid.version === 0) {
23+
if (cidBase && cidBase !== 'base58btc' && cid.version === 0) {
2424
cid = cid.toV1()
2525
}
2626

@@ -36,7 +36,7 @@ const mfsFlush = {
3636
},
3737
query: Joi.object().keys({
3838
arg: Joi.string(),
39-
cidBase: Joi.string().default('base58btc')
39+
cidBase: Joi.string()
4040
})
4141
}
4242
}

src/http/ls.js

Lines changed: 14 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -4,13 +4,17 @@ const Joi = require('@hapi/joi')
44
const {
55
PassThrough
66
} = require('stream')
7+
const toStream = require('it-to-stream')
8+
const all = require('it-all')
9+
10+
const mapEntry = (entry, options) => {
11+
options = options || {}
712

8-
const mapEntry = (entry) => {
913
const output = {
1014
Name: entry.name,
11-
Type: entry.type,
12-
Size: entry.size,
13-
Hash: entry.hash,
15+
Type: options.long ? entry.type : 0,
16+
Size: options.long ? entry.size || 0 : 0,
17+
Hash: entry.cid.toString(options.cidBase),
1418
Mode: entry.mode.toString(8).padStart(4, '0')
1519
}
1620

@@ -41,21 +45,18 @@ const mfsLs = {
4145

4246
if (stream) {
4347
const responseStream = await new Promise((resolve, reject) => {
44-
const readableStream = ipfs.files.lsReadableStream(arg, {
45-
long,
46-
cidBase
47-
})
48+
const readableStream = toStream.readable(ipfs.files.ls(arg), { objectMode: true })
4849

4950
const passThrough = new PassThrough()
5051

5152
readableStream.on('data', (entry) => {
5253
resolve(passThrough)
53-
passThrough.write(JSON.stringify(mapEntry(entry)) + '\n')
54+
passThrough.write(JSON.stringify(mapEntry(entry, { cidBase, long })) + '\n')
5455
})
5556

5657
readableStream.once('end', (entry) => {
5758
resolve(passThrough)
58-
passThrough.end(entry ? JSON.stringify(mapEntry(entry)) + '\n' : undefined)
59+
passThrough.end(entry ? JSON.stringify(mapEntry(entry, { cidBase, long })) + '\n' : undefined)
5960
})
6061

6162
readableStream.once('error', (err) => {
@@ -67,13 +68,10 @@ const mfsLs = {
6768
return h.response(responseStream).header('X-Stream-Output', '1')
6869
}
6970

70-
const files = await ipfs.files.ls(arg, {
71-
long,
72-
cidBase
73-
})
71+
const files = await all(ipfs.files.ls(arg))
7472

7573
return h.response({
76-
Entries: files.map(mapEntry)
74+
Entries: files.map(entry => mapEntry(entry, { cidBase, long }))
7775
})
7876
},
7977
options: {
@@ -85,7 +83,7 @@ const mfsLs = {
8583
query: Joi.object().keys({
8684
arg: Joi.string().default('/'),
8785
long: Joi.boolean().default(false),
88-
cidBase: Joi.string().default('base58btc'),
86+
cidBase: Joi.string(),
8987
stream: Joi.boolean().default(false)
9088
})
9189
.rename('l', 'long', {

src/http/read.js

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ const Joi = require('@hapi/joi')
44
const {
55
PassThrough
66
} = require('stream')
7+
const toStream = require('it-to-stream')
78

89
const mfsRead = {
910
method: 'POST',
@@ -19,10 +20,10 @@ const mfsRead = {
1920
} = request.query
2021

2122
const responseStream = await new Promise((resolve, reject) => {
22-
const stream = ipfs.files.readReadableStream(arg, {
23+
const stream = toStream.readable(ipfs.files.read(arg, {
2324
offset,
2425
length
25-
})
26+
}))
2627

2728
stream.once('data', (chunk) => {
2829
const passThrough = new PassThrough()

src/http/stat.js

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,15 +20,14 @@ const mfsStat = {
2020
const stats = await ipfs.files.stat(arg, {
2121
hash,
2222
size,
23-
withLocal,
24-
cidBase
23+
withLocal
2524
})
2625

2726
return h.response({
2827
Type: stats.type,
2928
Blocks: stats.blocks,
3029
Size: stats.size,
31-
Hash: stats.hash,
30+
Hash: stats.cid.toString(cidBase),
3231
CumulativeSize: stats.cumulativeSize,
3332
WithLocality: stats.withLocality,
3433
Local: stats.local,
@@ -49,7 +48,7 @@ const mfsStat = {
4948
hash: Joi.boolean().default(false),
5049
size: Joi.boolean().default(false),
5150
withLocal: Joi.boolean().default(false),
52-
cidBase: Joi.string().default('base58btc')
51+
cidBase: Joi.string()
5352
})
5453
}
5554
}

0 commit comments

Comments
 (0)