From 3f6f22f9d2042c16959510cc48bb053ac00e287e Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Tue, 25 Aug 2020 07:20:24 +0100 Subject: [PATCH] feat: store pins in datastore instead of a DAG (#2771) Adds a `.pins` datastore to `ipfs-repo` and uses that to store pins as cbor binary keyed by multihash. ### Format As stored in the datastore, each pin has several fields: ```javascript { codec: // optional Number, the codec from the CID that this multihash was pinned with, if omitted, treated as 'dag-pb' version: // optional Number, the version number from the CID that this multihash was pinned with, if omitted, treated as v0 depth: // Number Infinity = recursive pin, 0 = direct, 1+ = pinned to a depth comments: // optional String user-friendly description of the pin metadata: // optional Object, user-defined data for the pin } ``` Notes: `.codec` and `.version` are stored so we can recreate the original CID when listing pins. ### Metadata The intention is for us to be able to add extra fields that have technical meaning to the root of the object, and the user can store application-specific data in the `metadata` field. ### CLI ```console $ ipfs pin add bafyfoo --metadata key1=value1,key2=value2 $ ipfs pin add bafyfoo --metadata-format=json --metadata '{"key1":"value1","key2":"value2"}' $ ipfs pin list bafyfoo $ ipfs pin list -l CID Name Type Metadata bafyfoo My pin Recursive {"key1":"value1","key2":"value2"} $ ipfs pin metadata Qmfoo --format=json {"key1":"value1","key2":"value2"} ``` ### HTTP API * '/api/v0/pin/add' route adds new `metadata` argument, accepts a json string * '/api/v0/pin/metadata' returns metadata as json ### Core API * `ipfs.pin.addAll` accepts and returns an async iterator * `ipfs.pin.rmAll` accepts and returns an async iterator ```javascript // pass a cid or IPFS Path with options const { cid } = await ipfs.pin.add(new CID('/ipfs/Qmfoo'), { recursive: false, metadata: { key: 'value }, timeout: 2000 })) // pass an iterable of CIDs const [{ cid: cid1 }, { cid: cid2 }] = await all(ipfs.pin.addAll([ new CID('/ipfs/Qmfoo'), new CID('/ipfs/Qmbar') ], { timeout: '2s' })) // pass an iterable of objects with options const [{ cid: cid1 }, { cid: cid2 }] = await all(ipfs.pin.addAll([ { cid: new CID('/ipfs/Qmfoo'), recursive: true, comments: 'A recursive pin' }, { cid: new CID('/ipfs/Qmbar'), recursive: false, comments: 'A direct pin' } ], { timeout: '2s' })) ``` * ipfs.pin.rmAll accepts and returns an async generator (other input types are available) ```javascript // pass an IPFS Path or CID const { cid } = await ipfs.rm(new CID('/ipfs/Qmfoo/file.txt')) // pass options const { cid } = await all(ipfs.rm(new CID('/ipfs/Qmfoo'), { recursive: true })) // pass an iterable of CIDs or objects with options const [{ cid }] = await all(ipfs.rmAll([{ cid: new CID('/ipfs/Qmfoo'), recursive: true }])) ``` Bonus: Lets us pipe the output of one command into another: ```javascript await pipe( ipfs.pin.ls({ type: 'recursive' }), (source) => ipfs.pin.rmAll(source) ) // or await all(ipfs.pin.rmAll(ipfs.pin.ls({ type: 'recursive'}))) ``` BREAKING CHANGES: * pins are now stored in a datastore, a repo migration will occur on startup * All deps of this module now use Uint8Arrays in place of node Buffers --- package.json | 29 +++++++++++++------------ src/block/get.js | 3 +-- src/config/replace.js | 4 ++-- src/dag/get.js | 8 +++---- src/dht/find-peer.js | 3 +-- src/dht/get.js | 7 +++--- src/files/read.js | 5 +---- src/get.js | 6 +++--- src/index.js | 4 ++-- src/ls.js | 3 +-- src/object/data.js | 5 ++--- src/object/get.js | 6 +++--- src/object/links.js | 3 +-- src/object/patch/add-link.js | 3 +-- src/object/patch/append-data.js | 3 +-- src/object/patch/rm-link.js | 3 +-- src/object/patch/set-data.js | 3 +-- src/object/put.js | 15 +++++++------ src/object/stat.js | 3 +-- src/pin/add-all.js | 36 +++++++++++++++++++++++++++++++ src/pin/add.js | 29 +++++++++++-------------- src/pin/index.js | 4 +++- src/pin/ls.js | 22 ++++++++++++++++++- src/pin/rm-all.js | 38 +++++++++++++++++++++++++++++++++ src/pin/rm.js | 31 +++++++++++---------------- src/pubsub/subscribe.js | 16 ++++++++------ src/refs/index.js | 3 +-- test/commands.spec.js | 2 +- test/constructor.spec.js | 2 +- test/dag.spec.js | 16 +++++++------- test/diag.spec.js | 2 +- test/endpoint-config.spec.js | 2 +- test/exports.spec.js | 4 +--- test/files.spec.js | 6 +++--- test/get.spec.js | 5 ++--- test/interface.spec.js | 9 +++++++- test/key.spec.js | 2 +- test/lib.error-handler.spec.js | 2 +- test/log.spec.js | 6 +++--- test/node/custom-headers.js | 7 +++--- test/node/request-api.js | 6 +++--- test/node/swarm.js | 2 +- test/ping.spec.js | 2 +- test/repo.spec.js | 2 +- test/stats.spec.js | 2 +- test/sub-modules.spec.js | 2 +- 46 files changed, 230 insertions(+), 146 deletions(-) create mode 100644 src/pin/add-all.js create mode 100644 src/pin/rm-all.js diff --git a/package.json b/package.json index 614642698..9ef7a9d92 100644 --- a/package.json +++ b/package.json @@ -43,38 +43,39 @@ "abort-controller": "^3.0.0", "any-signal": "^1.1.0", "bignumber.js": "^9.0.0", - "buffer": "^5.6.0", - "cids": "^0.8.3", + "cids": "^1.0.0", "debug": "^4.1.0", "form-data": "^3.0.0", "ipfs-core-utils": "^0.3.2", "ipfs-utils": "^3.0.0", - "ipld-block": "^0.9.2", - "ipld-dag-cbor": "^0.16.0", - "ipld-dag-pb": "^0.19.0", - "ipld-raw": "^5.0.0", + "ipld-block": "^0.10.0", + "ipld-dag-cbor": "^0.17.0", + "ipld-dag-pb": "^0.20.0", + "ipld-raw": "^6.0.0", "iso-url": "^0.4.7", "it-last": "^1.0.1", + "it-map": "^1.0.2", "it-tar": "^1.2.2", "it-to-buffer": "^1.0.0", "it-to-stream": "^0.1.1", "merge-options": "^2.0.0", - "multiaddr": "^7.4.3", - "multiaddr-to-uri": "^5.1.0", - "multibase": "^1.0.1", - "multicodec": "^1.0.0", - "multihashes": "^1.0.1", + "multiaddr": "^8.0.0", + "multiaddr-to-uri": "^6.0.0", + "multibase": "^3.0.0", + "multicodec": "^2.0.0", + "multihashes": "^3.0.1", "nanoid": "^3.0.2", "node-fetch": "^2.6.0", "parse-duration": "^0.4.4", - "stream-to-it": "^0.2.1" + "stream-to-it": "^0.2.1", + "uint8arrays": "^1.1.0" }, "devDependencies": { - "aegir": "^23.0.0", + "aegir": "^26.0.0", "cross-env": "^7.0.0", "go-ipfs": "^0.6.0", "interface-ipfs-core": "^0.139.1", - "ipfsd-ctl": "^5.0.0", + "ipfsd-ctl": "^6.0.0", "it-all": "^1.0.1", "it-concat": "^1.0.0", "it-pipe": "^1.1.0", diff --git a/src/block/get.js b/src/block/get.js index 5564aef09..f39aa6d96 100644 --- a/src/block/get.js +++ b/src/block/get.js @@ -2,7 +2,6 @@ const Block = require('ipld-block') const CID = require('cids') -const { Buffer } = require('buffer') const configure = require('../lib/configure') const toUrlSearchParams = require('../lib/to-url-search-params') @@ -20,6 +19,6 @@ module.exports = configure(api => { headers: options.headers }) - return new Block(Buffer.from(await res.arrayBuffer()), cid) + return new Block(new Uint8Array(await res.arrayBuffer()), cid) } }) diff --git a/src/config/replace.js b/src/config/replace.js index 57fce31db..63ced0e20 100644 --- a/src/config/replace.js +++ b/src/config/replace.js @@ -1,6 +1,6 @@ 'use strict' -const { Buffer } = require('buffer') +const uint8ArrayFromString = require('uint8arrays/from-string') const multipartRequest = require('../lib/multipart-request') const configure = require('../lib/configure') const toUrlSearchParams = require('../lib/to-url-search-params') @@ -18,7 +18,7 @@ module.exports = configure(api => { signal, searchParams: toUrlSearchParams(options), ...( - await multipartRequest(Buffer.from(JSON.stringify(config)), controller, options.headers) + await multipartRequest(uint8ArrayFromString(JSON.stringify(config)), controller, options.headers) ) }) diff --git a/src/dag/get.js b/src/dag/get.js index 426a59a34..1094b9737 100644 --- a/src/dag/get.js +++ b/src/dag/get.js @@ -18,16 +18,16 @@ module.exports = configure((api, options) => { return async (cid, options = {}) => { const resolved = await dagResolve(cid, options) const block = await getBlock(resolved.cid, options) - const dagResolver = resolvers[block.cid.codec] + const dagResolver = resolvers[resolved.cid.codec] if (!dagResolver) { throw Object.assign( - new Error(`Missing IPLD format "${block.cid.codec}"`), - { missingMulticodec: cid.codec } + new Error(`Missing IPLD format "${resolved.cid.codec}"`), + { missingMulticodec: resolved.cid.codec } ) } - if (block.cid.codec === 'raw' && !resolved.remPath) { + if (resolved.cid.codec === 'raw' && !resolved.remPath) { resolved.remainderPath = '/' } diff --git a/src/dht/find-peer.js b/src/dht/find-peer.js index 43bd9ff35..40985143f 100644 --- a/src/dht/find-peer.js +++ b/src/dht/find-peer.js @@ -1,6 +1,5 @@ 'use strict' -const { Buffer } = require('buffer') const CID = require('cids') const multiaddr = require('multiaddr') const configure = require('../lib/configure') @@ -13,7 +12,7 @@ module.exports = configure(api => { timeout: options.timeout, signal: options.signal, searchParams: toUrlSearchParams({ - arg: `${Buffer.isBuffer(peerId) ? new CID(peerId) : peerId}`, + arg: `${peerId instanceof Uint8Array ? new CID(peerId) : peerId}`, ...options }), headers: options.headers diff --git a/src/dht/get.js b/src/dht/get.js index f1299e209..30ec158f9 100644 --- a/src/dht/get.js +++ b/src/dht/get.js @@ -1,9 +1,10 @@ 'use strict' -const { Buffer } = require('buffer') const configure = require('../lib/configure') const toUrlSearchParams = require('../lib/to-url-search-params') const { Value } = require('./response-types') +const uint8ArrayToString = require('uint8arrays/to-string') +const uint8ArrayFromString = require('uint8arrays/from-string') module.exports = configure(api => { return async function get (key, options = {}) { @@ -11,7 +12,7 @@ module.exports = configure(api => { timeout: options.timeout, signal: options.signal, searchParams: toUrlSearchParams({ - arg: Buffer.isBuffer(key) ? key.toString() : key, + arg: key instanceof Uint8Array ? uint8ArrayToString(key) : key, ...options }), headers: options.headers @@ -19,7 +20,7 @@ module.exports = configure(api => { for await (const message of res.ndjson()) { if (message.Type === Value) { - return Buffer.from(message.Extra, 'base64') + return uint8ArrayFromString(message.Extra, 'base64pad') } } diff --git a/src/files/read.js b/src/files/read.js index a85e33a1d..fb1fc048a 100644 --- a/src/files/read.js +++ b/src/files/read.js @@ -1,6 +1,5 @@ 'use strict' -const { Buffer } = require('buffer') const toIterable = require('stream-to-it/source') const configure = require('../lib/configure') const toUrlSearchParams = require('../lib/to-url-search-params') @@ -18,8 +17,6 @@ module.exports = configure(api => { headers: options.headers }) - for await (const chunk of toIterable(res.body)) { - yield Buffer.from(chunk) - } + yield * toIterable(res.body) } }) diff --git a/src/get.js b/src/get.js index 43bcd18e0..387d5fd79 100644 --- a/src/get.js +++ b/src/get.js @@ -1,10 +1,10 @@ 'use strict' const Tar = require('it-tar') -const { Buffer } = require('buffer') const CID = require('cids') const configure = require('./lib/configure') const toUrlSearchParams = require('./lib/to-url-search-params') +const map = require('it-map') module.exports = configure(api => { return async function * get (path, options = {}) { @@ -12,7 +12,7 @@ module.exports = configure(api => { timeout: options.timeout, signal: options.signal, searchParams: toUrlSearchParams({ - arg: `${Buffer.isBuffer(path) ? new CID(path) : path}`, + arg: `${path instanceof Uint8Array ? new CID(path) : path}`, ...options }), headers: options.headers @@ -28,7 +28,7 @@ module.exports = configure(api => { } else { yield { path: header.name, - content: body + content: map(body, (chunk) => chunk.slice()) // convert bl to Buffer/Uint8Array } } } diff --git a/src/index.js b/src/index.js index aeb82ee70..3d0eeeedd 100644 --- a/src/index.js +++ b/src/index.js @@ -1,6 +1,6 @@ 'use strict' /* eslint-env browser */ -const { Buffer } = require('buffer') + const CID = require('cids') const multiaddr = require('multiaddr') const multibase = require('multibase') @@ -56,6 +56,6 @@ function ipfsClient (options = {}) { } } -Object.assign(ipfsClient, { Buffer, CID, multiaddr, multibase, multicodec, multihash, globSource, urlSource }) +Object.assign(ipfsClient, { CID, multiaddr, multibase, multicodec, multihash, globSource, urlSource }) module.exports = ipfsClient diff --git a/src/ls.js b/src/ls.js index b448a0c2e..ff9cdbb57 100644 --- a/src/ls.js +++ b/src/ls.js @@ -1,6 +1,5 @@ 'use strict' -const { Buffer } = require('buffer') const CID = require('cids') const configure = require('./lib/configure') const toUrlSearchParams = require('./lib/to-url-search-params') @@ -11,7 +10,7 @@ module.exports = configure(api => { timeout: options.timeout, signal: options.signal, searchParams: toUrlSearchParams({ - arg: `${Buffer.isBuffer(path) ? new CID(path) : path}`, + arg: `${path instanceof Uint8Array ? new CID(path) : path}`, ...options }), headers: options.headers diff --git a/src/object/data.js b/src/object/data.js index 39446a993..1987a3d39 100644 --- a/src/object/data.js +++ b/src/object/data.js @@ -1,6 +1,5 @@ 'use strict' -const { Buffer } = require('buffer') const CID = require('cids') const configure = require('../lib/configure') const toUrlSearchParams = require('../lib/to-url-search-params') @@ -11,13 +10,13 @@ module.exports = configure(api => { timeout: options.timeout, signal: options.signal, searchParams: toUrlSearchParams({ - arg: `${Buffer.isBuffer(cid) ? new CID(cid) : cid}`, + arg: `${cid instanceof Uint8Array ? new CID(cid) : cid}`, ...options }), headers: options.headers }) const data = await res.arrayBuffer() - return Buffer.from(data) + return new Uint8Array(data, data.byteOffset, data.byteLength) } }) diff --git a/src/object/get.js b/src/object/get.js index 0ad60f7bf..4857fe724 100644 --- a/src/object/get.js +++ b/src/object/get.js @@ -1,10 +1,10 @@ 'use strict' -const { Buffer } = require('buffer') const CID = require('cids') const { DAGNode, DAGLink } = require('ipld-dag-pb') const configure = require('../lib/configure') const toUrlSearchParams = require('../lib/to-url-search-params') +const uint8ArrayFromString = require('uint8arrays/from-string') module.exports = configure(api => { return async (cid, options = {}) => { @@ -12,7 +12,7 @@ module.exports = configure(api => { timeout: options.timeout, signal: options.signal, searchParams: toUrlSearchParams({ - arg: `${Buffer.isBuffer(cid) ? new CID(cid) : cid}`, + arg: `${cid instanceof Uint8Array ? new CID(cid) : cid}`, dataEncoding: 'base64', ...options }), @@ -21,7 +21,7 @@ module.exports = configure(api => { const data = await res.json() return new DAGNode( - Buffer.from(data.Data, 'base64'), + uint8ArrayFromString(data.Data, 'base64pad'), (data.Links || []).map(l => new DAGLink(l.Name, l.Size, l.Hash)) ) } diff --git a/src/object/links.js b/src/object/links.js index abe71de54..97f2f5b33 100644 --- a/src/object/links.js +++ b/src/object/links.js @@ -1,6 +1,5 @@ 'use strict' -const { Buffer } = require('buffer') const CID = require('cids') const { DAGLink } = require('ipld-dag-pb') const configure = require('../lib/configure') @@ -12,7 +11,7 @@ module.exports = configure(api => { timeout: options.timeout, signal: options.signal, searchParams: toUrlSearchParams({ - arg: `${Buffer.isBuffer(cid) ? new CID(cid) : cid}`, + arg: `${cid instanceof Uint8Array ? new CID(cid) : cid}`, ...options }), headers: options.headers diff --git a/src/object/patch/add-link.js b/src/object/patch/add-link.js index f9f6eb8bf..ab8535406 100644 --- a/src/object/patch/add-link.js +++ b/src/object/patch/add-link.js @@ -1,6 +1,5 @@ 'use strict' -const { Buffer } = require('buffer') const CID = require('cids') const configure = require('../../lib/configure') const toUrlSearchParams = require('../../lib/to-url-search-params') @@ -12,7 +11,7 @@ module.exports = configure(api => { signal: options.signal, searchParams: toUrlSearchParams({ arg: [ - `${Buffer.isBuffer(cid) ? new CID(cid) : cid}`, + `${cid instanceof Uint8Array ? new CID(cid) : cid}`, dLink.Name || dLink.name || '', (dLink.Hash || dLink.cid || '').toString() || null ], diff --git a/src/object/patch/append-data.js b/src/object/patch/append-data.js index 0ef14f9a5..c33490750 100644 --- a/src/object/patch/append-data.js +++ b/src/object/patch/append-data.js @@ -1,6 +1,5 @@ 'use strict' -const { Buffer } = require('buffer') const CID = require('cids') const multipartRequest = require('../../lib/multipart-request') const configure = require('../../lib/configure') @@ -18,7 +17,7 @@ module.exports = configure(api => { timeout: options.timeout, signal, searchParams: toUrlSearchParams({ - arg: `${Buffer.isBuffer(cid) ? new CID(cid) : cid}`, + arg: `${cid instanceof Uint8Array ? new CID(cid) : cid}`, ...options }), ...( diff --git a/src/object/patch/rm-link.js b/src/object/patch/rm-link.js index 89f02b77b..3a58238c0 100644 --- a/src/object/patch/rm-link.js +++ b/src/object/patch/rm-link.js @@ -1,6 +1,5 @@ 'use strict' -const { Buffer } = require('buffer') const CID = require('cids') const configure = require('../../lib/configure') const toUrlSearchParams = require('../../lib/to-url-search-params') @@ -12,7 +11,7 @@ module.exports = configure(api => { signal: options.signal, searchParams: toUrlSearchParams({ arg: [ - `${Buffer.isBuffer(cid) ? new CID(cid) : cid}`, + `${cid instanceof Uint8Array ? new CID(cid) : cid}`, dLink.Name || dLink.name || null ], ...options diff --git a/src/object/patch/set-data.js b/src/object/patch/set-data.js index 9609dff3a..59c3394b5 100644 --- a/src/object/patch/set-data.js +++ b/src/object/patch/set-data.js @@ -1,6 +1,5 @@ 'use strict' -const { Buffer } = require('buffer') const CID = require('cids') const multipartRequest = require('../../lib/multipart-request') const configure = require('../../lib/configure') @@ -19,7 +18,7 @@ module.exports = configure(api => { signal, searchParams: toUrlSearchParams({ arg: [ - `${Buffer.isBuffer(cid) ? new CID(cid) : cid}` + `${cid instanceof Uint8Array ? new CID(cid) : cid}` ], ...options }), diff --git a/src/object/put.js b/src/object/put.js index 34bf8eebe..c9e46898a 100644 --- a/src/object/put.js +++ b/src/object/put.js @@ -2,12 +2,13 @@ const CID = require('cids') const { DAGNode } = require('ipld-dag-pb') -const { Buffer } = require('buffer') const multipartRequest = require('../lib/multipart-request') const configure = require('../lib/configure') const toUrlSearchParams = require('../lib/to-url-search-params') const anySignal = require('any-signal') const AbortController = require('abort-controller') +const unit8ArrayToString = require('uint8arrays/to-string') +const uint8ArrayFromString = require('uint8arrays/from-string') module.exports = configure(api => { return async (obj, options = {}) => { @@ -16,16 +17,16 @@ module.exports = configure(api => { Links: [] } - if (Buffer.isBuffer(obj)) { + if (obj instanceof Uint8Array) { if (!options.enc) { tmpObj = { - Data: obj.toString(), + Data: unit8ArrayToString(obj), Links: [] } } } else if (DAGNode.isDAGNode(obj)) { tmpObj = { - Data: obj.Data.toString(), + Data: unit8ArrayToString(obj.Data), Links: obj.Links.map(l => ({ Name: l.Name, Hash: l.Hash.toString(), @@ -33,18 +34,18 @@ module.exports = configure(api => { })) } } else if (typeof obj === 'object') { - tmpObj.Data = obj.Data.toString() + tmpObj.Data = unit8ArrayToString(obj.Data) tmpObj.Links = obj.Links } else { throw new Error('obj not recognized') } let buf - if (Buffer.isBuffer(obj) && options.enc) { + if (obj instanceof Uint8Array && options.enc) { buf = obj } else { options.enc = 'json' - buf = Buffer.from(JSON.stringify(tmpObj)) + buf = uint8ArrayFromString(JSON.stringify(tmpObj)) } // allow aborting requests on body errors diff --git a/src/object/stat.js b/src/object/stat.js index 1b8bfeb87..294a1c222 100644 --- a/src/object/stat.js +++ b/src/object/stat.js @@ -1,6 +1,5 @@ 'use strict' -const { Buffer } = require('buffer') const CID = require('cids') const configure = require('../lib/configure') const toUrlSearchParams = require('../lib/to-url-search-params') @@ -11,7 +10,7 @@ module.exports = configure(api => { timeout: options.timeout, signal: options.signal, searchParams: toUrlSearchParams({ - arg: `${Buffer.isBuffer(cid) ? new CID(cid) : cid}`, + arg: `${cid instanceof Uint8Array ? new CID(cid) : cid}`, ...options }), headers: options.headers diff --git a/src/pin/add-all.js b/src/pin/add-all.js new file mode 100644 index 000000000..e49f386d4 --- /dev/null +++ b/src/pin/add-all.js @@ -0,0 +1,36 @@ +'use strict' + +const CID = require('cids') +const configure = require('../lib/configure') +const normaliseInput = require('ipfs-core-utils/src/pins/normalise-input') +const toUrlSearchParams = require('../lib/to-url-search-params') + +module.exports = configure(api => { + return async function * addAll (source, options = {}) { + for await (const { path, recursive, metadata } of normaliseInput(source)) { + const res = await api.post('pin/add', { + timeout: options.timeout, + signal: options.signal, + searchParams: toUrlSearchParams({ + ...options, + arg: path, + recursive, + metadata: metadata ? JSON.stringify(metadata) : undefined, + stream: true + }), + headers: options.headers + }) + + for await (const pin of res.ndjson()) { + if (pin.Pins) { // non-streaming response + for (const cid of pin.Pins) { + yield new CID(cid) + } + continue + } + + yield new CID(pin) + } + } + } +}) diff --git a/src/pin/add.js b/src/pin/add.js index 6180f5d3d..730301016 100644 --- a/src/pin/add.js +++ b/src/pin/add.js @@ -1,23 +1,18 @@ 'use strict' -const CID = require('cids') +const addAll = require('./add-all') +const last = require('it-last') const configure = require('../lib/configure') -const toUrlSearchParams = require('../lib/to-url-search-params') -module.exports = configure(api => { - return async (paths, options = {}) => { - paths = Array.isArray(paths) ? paths : [paths] +module.exports = (options) => { + const all = addAll(options) - const res = await (await api.post('pin/add', { - timeout: options.timeout, - signal: options.signal, - searchParams: toUrlSearchParams({ - arg: paths.map(path => `${path}`), + return configure(() => { + return async function add (path, options = {}) { // eslint-disable-line require-await + return last(all({ + path, ...options - }), - headers: options.headers - })).json() - - return (res.Pins || []).map(cid => ({ cid: new CID(cid) })) - } -}) + }, options)) + } + })(options) +} diff --git a/src/pin/index.js b/src/pin/index.js index ad43057ec..39f7b5ecc 100644 --- a/src/pin/index.js +++ b/src/pin/index.js @@ -2,6 +2,8 @@ module.exports = config => ({ add: require('./add')(config), + addAll: require('./add-all')(config), + ls: require('./ls')(config), rm: require('./rm')(config), - ls: require('./ls')(config) + rmAll: require('./rm-all')(config) }) diff --git a/src/pin/ls.js b/src/pin/ls.js index 5133e3a56..c8ed0d136 100644 --- a/src/pin/ls.js +++ b/src/pin/ls.js @@ -4,6 +4,19 @@ const CID = require('cids') const configure = require('../lib/configure') const toUrlSearchParams = require('../lib/to-url-search-params') +function toPin (type, cid, metadata) { + const pin = { + type, + cid: new CID(cid) + } + + if (metadata) { + pin.metadata = metadata + } + + return pin +} + module.exports = configure(api => { return async function * ls (options = {}) { if (options.paths) { @@ -22,7 +35,14 @@ module.exports = configure(api => { }) for await (const pin of res.ndjson()) { - yield { cid: new CID(pin.Cid), type: pin.Type } + if (pin.Keys) { // non-streaming response + for (const cid of Object.keys(pin.Keys)) { + yield toPin(pin.Keys[cid].Type, cid, pin.Keys[cid].Metadata) + } + return + } + + yield toPin(pin.Type, pin.Cid, pin.Metadata) } } }) diff --git a/src/pin/rm-all.js b/src/pin/rm-all.js new file mode 100644 index 000000000..4e6445621 --- /dev/null +++ b/src/pin/rm-all.js @@ -0,0 +1,38 @@ +'use strict' + +const CID = require('cids') +const configure = require('../lib/configure') +const normaliseInput = require('ipfs-core-utils/src/pins/normalise-input') +const toUrlSearchParams = require('../lib/to-url-search-params') + +module.exports = configure(api => { + return async function * rmAll (source, options = {}) { + options = options || {} + + for await (const { path, recursive } of normaliseInput(source)) { + const searchParams = new URLSearchParams(options.searchParams) + searchParams.append('arg', `${path}`) + + if (recursive != null) searchParams.set('recursive', recursive) + + const res = await api.post('pin/rm', { + timeout: options.timeout, + signal: options.signal, + headers: options.headers, + searchParams: toUrlSearchParams({ + ...options, + arg: `${path}`, + recursive + }) + }) + + for await (const pin of res.ndjson()) { + if (pin.Pins) { // non-streaming response + yield * pin.Pins.map(cid => new CID(cid)) + continue + } + yield new CID(pin) + } + } + } +}) diff --git a/src/pin/rm.js b/src/pin/rm.js index a7d9b5e73..34f3daf06 100644 --- a/src/pin/rm.js +++ b/src/pin/rm.js @@ -1,23 +1,18 @@ 'use strict' -const CID = require('cids') +const rmAll = require('./rm-all') +const last = require('it-last') const configure = require('../lib/configure') -const toUrlSearchParams = require('../lib/to-url-search-params') -module.exports = configure(api => { - return async (path, options = {}) => { - const res = await api.post('pin/rm', { - timeout: options.timeout, - signal: options.signal, - searchParams: toUrlSearchParams({ - arg: `${path}`, - ...options - }), - headers: options.headers - }) - - const data = await res.json() +module.exports = (options) => { + const all = rmAll(options) - return (data.Pins || []).map(cid => ({ cid: new CID(cid) })) - } -}) + return configure(() => { + return async function rm (path, options = {}) { // eslint-disable-line require-await + return last(all({ + path, + ...options + }, options)) + } + })(options) +} diff --git a/src/pubsub/subscribe.js b/src/pubsub/subscribe.js index fdf4d24ad..c62fc8c8c 100644 --- a/src/pubsub/subscribe.js +++ b/src/pubsub/subscribe.js @@ -1,7 +1,7 @@ 'use strict' -const multibase = require('multibase') -const { Buffer } = require('buffer') +const uint8ArrayFromString = require('uint8arrays/from-string') +const uint8ArrayToString = require('uint8arrays/to-string') const log = require('debug')('ipfs-http-client:pubsub:subscribe') const SubscriptionTracker = require('./subscription-tracker') const configure = require('../lib/configure') @@ -22,7 +22,7 @@ module.exports = configure((api, options) => { const ffWorkaround = setTimeout(async () => { log(`Publishing empty message to "${topic}" to resolve subscription request`) try { - await publish(topic, Buffer.alloc(0), options) + await publish(topic, new Uint8Array(0), options) } catch (err) { log('Failed to publish empty message', err) } @@ -59,10 +59,14 @@ async function readMessages (msgStream, { onMessage, onEnd, onError }) { try { for await (const msg of msgStream) { try { + if (!msg.from) { + continue + } + onMessage({ - from: multibase.encode('base58btc', Buffer.from(msg.from, 'base64')).toString().slice(1), - data: Buffer.from(msg.data, 'base64'), - seqno: Buffer.from(msg.seqno, 'base64'), + from: uint8ArrayToString(uint8ArrayFromString(msg.from, 'base64pad'), 'base58btc'), + data: uint8ArrayFromString(msg.data, 'base64pad'), + seqno: uint8ArrayFromString(msg.seqno, 'base64pad'), topicIDs: msg.topicIDs }) } catch (err) { diff --git a/src/refs/index.js b/src/refs/index.js index c624434c7..b21560ee5 100644 --- a/src/refs/index.js +++ b/src/refs/index.js @@ -1,6 +1,5 @@ 'use strict' -const { Buffer } = require('buffer') const CID = require('cids') const toCamel = require('../lib/object-to-camel') const configure = require('../lib/configure') @@ -16,7 +15,7 @@ module.exports = configure((api, options) => { timeout: options.timeout, signal: options.signal, searchParams: toUrlSearchParams({ - arg: args.map(arg => `${Buffer.isBuffer(arg) ? new CID(arg) : arg}`), + arg: args.map(arg => `${arg instanceof Uint8Array ? new CID(arg) : arg}`), ...options }), headers: options.headers, diff --git a/test/commands.spec.js b/test/commands.spec.js index 96f932b92..9c7de185c 100644 --- a/test/commands.spec.js +++ b/test/commands.spec.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const f = require('./utils/factory')() describe('.commands', function () { diff --git a/test/constructor.spec.js b/test/constructor.spec.js index e0d188d0a..7cca6332b 100644 --- a/test/constructor.spec.js +++ b/test/constructor.spec.js @@ -2,7 +2,7 @@ 'use strict' const multiaddr = require('multiaddr') -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const f = require('./utils/factory')() const ipfsClient = require('../src/index.js') const globalThis = require('ipfs-utils/src/globalthis') diff --git a/test/dag.spec.js b/test/dag.spec.js index 5b33a6637..a5f861aeb 100644 --- a/test/dag.spec.js +++ b/test/dag.spec.js @@ -3,8 +3,8 @@ 'use strict' -const { Buffer } = require('buffer') -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const uint8ArrayFromString = require('uint8arrays/from-string') +const { expect } = require('aegir/utils/chai') const ipldDagPb = require('ipld-dag-pb') const { DAGNode } = ipldDagPb const CID = require('cids') @@ -22,7 +22,7 @@ describe('.dag', function () { after(() => f.clean()) it('should be able to put and get a DAG node with format dag-pb', async () => { - const data = Buffer.from('some data') + const data = uint8ArrayFromString('some data') const node = new DAGNode(data) let cid = await ipfs.dag.put(node, { format: 'dag-pb', hashAlg: 'sha2-256' }) @@ -50,7 +50,7 @@ describe('.dag', function () { }) it('should be able to put and get a DAG node with format raw', async () => { - const node = Buffer.from('some data') + const node = uint8ArrayFromString('some data') let cid = await ipfs.dag.put(node, { format: 'raw', hashAlg: 'sha2-256' }) expect(cid.codec).to.equal('raw') @@ -63,7 +63,7 @@ describe('.dag', function () { }) it('should error when missing DAG resolver for multicodec from requested CID', async () => { - const block = await ipfs.block.put(Buffer.from([0, 1, 2, 3]), { + const block = await ipfs.block.put(Uint8Array.from([0, 1, 2, 3]), { cid: new CID('z8mWaJ1dZ9fH5EetPuRsj8jj26pXsgpsr') }) @@ -71,7 +71,7 @@ describe('.dag', function () { }) it('should error when putting node with esoteric format', () => { - const node = Buffer.from('some data') + const node = uint8ArrayFromString('some data') return expect(ipfs.dag.put(node, { format: 'git-raw', hashAlg: 'sha2-256' })).to.eventually.be.rejectedWith(/Format unsupported/) }) @@ -92,7 +92,7 @@ describe('.dag', function () { } }) - const node = Buffer.from('some data') + const node = uint8ArrayFromString('some data') // error is from go-ipfs, this means the client serialized it ok await expect(ipfs2.dag.put(node, { format: 'git-raw', hashAlg: 'sha2-256' })).to.eventually.be.rejectedWith(/no parser for format "git-raw"/) @@ -118,7 +118,7 @@ describe('.dag', function () { hashAlg: 'sha2-256' }) - const dagPbNode = new DAGNode(Buffer.alloc(0), [], 0) + const dagPbNode = new DAGNode(new Uint8Array(0), [], 0) const cid2 = await ipfs2.dag.put(dagPbNode, { format: 'dag-pb', hashAlg: 'sha2-256' diff --git a/test/diag.spec.js b/test/diag.spec.js index 1c78916cf..60676d9a1 100644 --- a/test/diag.spec.js +++ b/test/diag.spec.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const f = require('./utils/factory')() describe('.diag', function () { diff --git a/test/endpoint-config.spec.js b/test/endpoint-config.spec.js index f1719d895..1c019fb0f 100644 --- a/test/endpoint-config.spec.js +++ b/test/endpoint-config.spec.js @@ -2,7 +2,7 @@ /* eslint max-nested-callbacks: ["error", 8] */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const ipfsClient = require('../src') describe('.getEndpointConfig', () => { diff --git a/test/exports.spec.js b/test/exports.spec.js index c3ddff17d..1cc99c946 100644 --- a/test/exports.spec.js +++ b/test/exports.spec.js @@ -2,18 +2,16 @@ 'use strict' const CID = require('cids') -const { Buffer } = require('buffer') const multiaddr = require('multiaddr') const multibase = require('multibase') const multicodec = require('multicodec') const multihash = require('multihashes') -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const IpfsHttpClient = require('../') describe('exports', () => { it('should export the expected types and utilities', () => { - expect(IpfsHttpClient.Buffer).to.equal(Buffer) expect(IpfsHttpClient.CID).to.equal(CID) expect(IpfsHttpClient.multiaddr).to.equal(multiaddr) expect(IpfsHttpClient.multibase).to.equal(multibase) diff --git a/test/files.spec.js b/test/files.spec.js index 9ad1be1ff..10e2d1c84 100644 --- a/test/files.spec.js +++ b/test/files.spec.js @@ -2,8 +2,8 @@ 'use strict' -const { Buffer } = require('buffer') -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const uint8ArrayFromString = require('uint8arrays/from-string') +const { expect } = require('aegir/utils/chai') const f = require('./utils/factory')() describe('.add', function () { @@ -18,7 +18,7 @@ describe('.add', function () { after(() => f.clean()) it('should ignore metadata until https://github.com/ipfs/go-ipfs/issues/6920 is implemented', async () => { - const data = Buffer.from('some data') + const data = uint8ArrayFromString('some data') const result = await ipfs.add(data, { mode: 0o600, mtime: { diff --git a/test/get.spec.js b/test/get.spec.js index dd2513c45..4f3044e84 100644 --- a/test/get.spec.js +++ b/test/get.spec.js @@ -3,9 +3,8 @@ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const loadFixture = require('aegir/fixtures') -const { Buffer } = require('buffer') const all = require('it-all') const concat = require('it-concat') @@ -67,7 +66,7 @@ describe('.get (specific go-ipfs features)', function () { const path = `${subdir}/${filename}` const files = await all(ipfs.addAll([{ path, - content: Buffer.from(path) + content: path }])) expect(files[2].cid.toString()).to.equal(expectedCid) diff --git a/test/interface.spec.js b/test/interface.spec.js index 9e367c160..451789311 100644 --- a/test/interface.spec.js +++ b/test/interface.spec.js @@ -550,7 +550,14 @@ describe('interface-ipfs-core tests', () => { ] }) - tests.pin(commonFactory) + tests.pin(commonFactory, { + skip: [ + { + name: 'should list pins with metadata', + reason: 'not implemented in go-ipfs' + } + ] + }) tests.ping(commonFactory, { skip: [ diff --git a/test/key.spec.js b/test/key.spec.js index eb53b8293..95028c744 100644 --- a/test/key.spec.js +++ b/test/key.spec.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const f = require('./utils/factory')() describe('.key', function () { diff --git a/test/lib.error-handler.spec.js b/test/lib.error-handler.spec.js index 9804ccdd4..7e7349861 100644 --- a/test/lib.error-handler.spec.js +++ b/test/lib.error-handler.spec.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const throwsAsync = require('./utils/throws-async') const { errorHandler, HTTPError } = require('../src/lib/core') diff --git a/test/log.spec.js b/test/log.spec.js index e41ea9b03..814ff4488 100644 --- a/test/log.spec.js +++ b/test/log.spec.js @@ -2,8 +2,8 @@ /* eslint max-nested-callbacks: ["error", 8] */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') -const { Buffer } = require('buffer') +const { expect } = require('aegir/utils/chai') +const uint8ArrayFromString = require('uint8arrays/from-string') const f = require('./utils/factory')() describe('.log', function () { @@ -20,7 +20,7 @@ describe('.log', function () { it('.log.tail', async () => { const i = setInterval(async () => { try { - await ipfs.add(Buffer.from('just adding some data to generate logs')) + await ipfs.add(uint8ArrayFromString('just adding some data to generate logs')) } catch (_) { // this can error if the test has finished and we're shutting down the node } diff --git a/test/node/custom-headers.js b/test/node/custom-headers.js index 98d3f1a68..2d0df7a84 100644 --- a/test/node/custom-headers.js +++ b/test/node/custom-headers.js @@ -2,8 +2,9 @@ 'use strict' const { isNode } = require('ipfs-utils/src/env') -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const ipfsClient = require('../../src') +const uint8ArrayFromString = require('uint8arrays/from-string') function startServer (fn) { let headersResolve @@ -59,7 +60,7 @@ describe('custom headers', function () { }) it('multipart API calls', async () => { - const headers = await startServer(() => ipfs.files.write('/foo/bar', Buffer.from('derp'), { + const headers = await startServer(() => ipfs.files.write('/foo/bar', uint8ArrayFromString('derp'), { create: true })) @@ -88,7 +89,7 @@ describe('custom headers', function () { }) it('multipart API calls', async () => { - const headers = await startServer(() => ipfs.files.write('/foo/bar', Buffer.from('derp'), { + const headers = await startServer(() => ipfs.files.write('/foo/bar', uint8ArrayFromString('derp'), { create: true, headers: { authorization: 'Bearer OLOY' diff --git a/test/node/request-api.js b/test/node/request-api.js index 422b0f0d7..ed91c6157 100644 --- a/test/node/request-api.js +++ b/test/node/request-api.js @@ -1,8 +1,8 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') -const { Buffer } = require('buffer') +const { expect } = require('aegir/utils/chai') +const uint8ArrayFromString = require('uint8arrays/from-string') const ipfsClient = require('../../src/index.js') describe('\'deal with HTTP weirdness\' tests', () => { @@ -40,7 +40,7 @@ describe('trailer headers', () => { server.listen(6001, () => { const ipfs = ipfsClient('/ip4/127.0.0.1/tcp/6001') /* eslint-disable */ - ipfs.add(Buffer.from('Hello there!'), (err, res) => { + ipfs.add(uint8ArrayFromString('Hello there!'), (err, res) => { // TODO: error's are not being correctly // propagated with Trailer headers yet // expect(err).to.exist() diff --git a/test/node/swarm.js b/test/node/swarm.js index 5f4d01643..c437caabf 100644 --- a/test/node/swarm.js +++ b/test/node/swarm.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const nock = require('nock') const ipfsClient = require('../../src') diff --git a/test/ping.spec.js b/test/ping.spec.js index b464062bb..46f4e4588 100644 --- a/test/ping.spec.js +++ b/test/ping.spec.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const all = require('it-all') const f = require('./utils/factory')() diff --git a/test/repo.spec.js b/test/repo.spec.js index 7a1b643d7..bac770858 100644 --- a/test/repo.spec.js +++ b/test/repo.spec.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const f = require('./utils/factory')() describe('.repo', function () { diff --git a/test/stats.spec.js b/test/stats.spec.js index 6b1530bee..abae59418 100644 --- a/test/stats.spec.js +++ b/test/stats.spec.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') const all = require('it-all') const f = require('./utils/factory')() diff --git a/test/sub-modules.spec.js b/test/sub-modules.spec.js index a5db456df..9b939d1e5 100644 --- a/test/sub-modules.spec.js +++ b/test/sub-modules.spec.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { expect } = require('interface-ipfs-core/src/utils/mocha') +const { expect } = require('aegir/utils/chai') describe('submodules', () => { it('bitswap', () => {