feat: store pins in datastore instead of a DAG

examples/custom-ipfs-repo/package.json

@@ -12,7 +12,7 @@
   "dependencies": {
     "datastore-fs": "^0.9.1",
     "ipfs": "^0.41.0",
-    "ipfs-repo": "^0.30.1",
+    "ipfs-repo": "github:ipfs/js-ipfs-repo#store-pins-in-datastore",
     "it-all": "^1.0.1"
   },
   "devDependencies": {

packages/interface-ipfs-core/SPEC/PIN.md

@@ -8,22 +8,25 @@
 
 > Adds an IPFS object to the pinset and also stores it to the IPFS repo. pinset is the set of hashes currently pinned (not gc'able).
 
-##### `ipfs.pin.add(hash, [options])`
+##### `ipfs.pin.add(source, [options])`
 
 Where:
 
-- `hash` is an IPFS multihash.
+- `source` is a [CID], an IPFS path or an object of the form `{ cid: CID, recursive: Boolean, comments: String }` or `{ path: String, recursive: Boolean, comments: String }` or an array or (async) iterable that yields any of the previous types
+  - Where this is an object, the fields mean the following:
+    - `cid`: The CID to pin
+    - `recursive`: Where the block that the CID resolves to contains links to other blocks, pin those too
+    - `comments`: A human readable string that can be used to add context to a pin
 - `options` is an object that can contain the following keys
-  - `recursive` (`boolean`) - Recursively pin the object linked. Type: bool. Default: `true`
   - `timeout` (`number`|`string`) - Throw an error if the request does not complete within the specified milliseconds timeout. If `timeout` is a string, the value is parsed as a [human readable duration](https://www.npmjs.com/package/parse-duration). There is no timeout by default.
 
 **Returns**
 
 | Type | Description |
 | -------- | -------- |
-| `Promise<{ cid: CID }>` | An array of objects that represent the files that were pinned |
+| `AsyncIterable<{ cid: CID }>` | An async iterable that yields objects containing the CIDs that were pinned |
 
-an array of objects is returned, each of the form:
+Each yielded object has the form:
 
 ```JavaScript
 {
@@ -34,10 +37,10 @@ an array of objects is returned, each of the form:
 **Example:**
 
 ```JavaScript
-const pinset = await ipfs.pin.add('QmWATWQ7fVPP2EFGu71UkfnqhYXDYH566qy47CnJDgvs8u')
-console.log(pinset)
-// Logs:
-// [ { cid: CID('QmWATWQ7fVPP2EFGu71UkfnqhYXDYH566qy47CnJDgvs8u') } ]
+for await (const pin of ipfs.pin.add(new CID('QmWATWQ7fVPP2EFGu71UkfnqhYXDYH566qy47CnJDgvs8u'))) {
+  console.log(pinset)
+}
+// { cid: CID('QmWATWQ7fVPP2EFGu71UkfnqhYXDYH566qy47CnJDgvs8u') }
 ```
 
 A great source of [examples][] can be found in the tests for this API.
@@ -77,26 +80,26 @@ A great source of [examples][] can be found in the tests for this API.
 
 > Remove a hash from the pinset
 
-##### `ipfs.pin.rm(hash, [options])`
+##### `ipfs.pin.rm(source, [options])`
 
 Where:
-- `hash` is a multihash.
+- `source` is a [CID], an IPFS path or an object of the form `{ cid: CID, recursive: Boolean}` or `{ path: String, recursive: Boolean }` or an array or (async) iterable that yields any of the previous types
 - `options` is an object that can contain the following keys
-  - 'recursive' - Recursively unpin the object linked. Type: bool. Default: `true`
 
 **Returns**
 
 | Type | Description |
 | -------- | -------- |
-| `Promise<{ cid: CID }>` | An array of unpinned objects |
+| `AsyncIterable<{ cid: CID }>` | An async iterable that yields objects containing the CIDs that were unpinned |
 
 **Example:**
 
 ```JavaScript
-const pinset = await ipfs.pin.rm('QmWATWQ7fVPP2EFGu71UkfnqhYXDYH566qy47CnJDgvs8u')
-console.log(pinset)
+for await (const unpinned of ipfs.pin.rm(new CID('QmWATWQ7fVPP2EFGu71UkfnqhYXDYH566qy47CnJDgvs8u'))) {
+  console.log(unpinned)
+}
 // prints the hashes that were unpinned
-// [ { cid: CID('QmWATWQ7fVPP2EFGu71UkfnqhYXDYH566qy47CnJDgvs8u') } ]
+// { cid: CID('QmWATWQ7fVPP2EFGu71UkfnqhYXDYH566qy47CnJDgvs8u') }
 ```
 
 A great source of [examples][] can be found in the tests for this API.

packages/interface-ipfs-core/package.json

@@ -45,6 +45,7 @@
     "is-ipfs": "^0.6.1",
     "it-all": "^1.0.1",
     "it-concat": "^1.0.0",
+    "it-drain": "^1.0.0",
     "it-last": "^1.0.1",
     "it-pushable": "^1.3.1",
     "multiaddr": "^7.2.1",

packages/interface-ipfs-core/src/block/rm.js

@@ -4,6 +4,8 @@
 const { getDescribe, getIt, expect } = require('../utils/mocha')
 const hat = require('hat')
 const all = require('it-all')
+const last = require('it-last')
+const drain = require('it-drain')
 
 /** @typedef { import("ipfsd-ctl/src/factory") } Factory */
 /**
@@ -142,13 +144,12 @@ module.exports = (common, options) => {
         format: 'raw',
         hashAlg: 'sha2-256'
       })
-      await ipfs.pin.add(cid.toString())
+      await drain(ipfs.pin.add(cid))
 
-      const result = await all(ipfs.block.rm(cid))
+      const result = await last(ipfs.block.rm(cid))
 
-      expect(result).to.be.an('array').and.to.have.lengthOf(1)
-      expect(result[0]).to.have.property('error')
-      expect(result[0].error.message).to.include('pinned')
+      expect(result).to.have.property('error').that.is.an('Error')
+        .with.property('message').that.includes('pinned')
     })
   })
 }

packages/interface-ipfs-core/src/pin/add.js

@@ -1,9 +1,13 @@
 /* eslint-env mocha */
 'use strict'
 
-const { fixtures } = require('./utils')
+const { fixtures, clearPins, expectPinned, pinTypes } = require('./utils')
 const { getDescribe, getIt, expect } = require('../utils/mocha')
 const all = require('it-all')
+const drain = require('it-drain')
+const {
+  DAGNode
+} = require('ipld-dag-pb')
 
 /** @typedef { import("ipfsd-ctl/src/factory") } Factory */
 /**
@@ -20,16 +24,230 @@ module.exports = (common, options) => {
     let ipfs
     before(async () => {
       ipfs = (await common.spawn()).api
+
       await Promise.all(fixtures.files.map(file => {
         return all(ipfs.add(file.data, { pin: false }))
       }))
+
+      await all(
+        ipfs.add(fixtures.directory.files.map(
+          file => ({
+            path: file.path,
+            content: file.data
+          })
+        ), {
+          pin: false
+        })
+      )
     })
 
     after(() => common.clean())
 
-    it('should add a pin', async () => {
-      const pinset = await ipfs.pin.add(fixtures.files[0].cid, { recursive: false })
-      expect(pinset.map(p => p.cid.toString())).to.include(fixtures.files[0].cid)
+    beforeEach(() => {
+      return clearPins(ipfs)
+    })
+
+    async function testAddInput (source) {
+      const pinset = await all(ipfs.pin.add(source))
+
+      expect(pinset.map(p => p.cid)).to.have.deep.members([
+        fixtures.files[0].cid,
+        fixtures.files[1].cid
+      ])
+    }
+
+    it('should add a CID and return the added CID', async () => {
+      const pinset = await all(ipfs.pin.add(fixtures.files[0].cid))
+      expect(pinset.map(p => p.cid)).to.deep.include(fixtures.files[0].cid)
+    })
+
+    it('should add a pin with options and return the added CID', async () => {
+      const pinset = await all(ipfs.pin.add({
+        cid: fixtures.files[0].cid,
+        recursive: false
+      }))
+      expect(pinset.map(p => p.cid)).to.deep.include(fixtures.files[0].cid)
+    })
+
+    it('should add an array of CIDs', () => {
+      return testAddInput([
+        fixtures.files[0].cid,
+        fixtures.files[1].cid
+      ])
+    })
+
+    it('should add a generator of CIDs', () => {
+      return testAddInput(function * () {
+        yield fixtures.files[0].cid
+        yield fixtures.files[1].cid
+      }())
+    })
+
+    it('should add an async generator of CIDs', () => {
+      return testAddInput(async function * () { // eslint-disable-line require-await
+        yield fixtures.files[0].cid
+        yield fixtures.files[1].cid
+      }())
+    })
+
+    it('should add an array of pins with options', () => {
+      return testAddInput([
+        {
+          cid: fixtures.files[0].cid,
+          recursive: false
+        },
+        {
+          cid: fixtures.files[1].cid,
+          recursive: true
+        }
+      ])
+    })
+
+    it('should add a generator of pins with options', () => {
+      return testAddInput(function * () {
+        yield {
+          cid: fixtures.files[0].cid,
+          recursive: false
+        }
+        yield {
+          cid: fixtures.files[1].cid,
+          recursive: true
+        }
+      }())
+    })
+
+    it('should add an async generator of pins with options', () => {
+      return testAddInput(async function * () { // eslint-disable-line require-await
+        yield {
+          cid: fixtures.files[0].cid,
+          recursive: false
+        }
+        yield {
+          cid: fixtures.files[1].cid,
+          recursive: true
+        }
+      }())
+    })
+
+    it('should add recursively', async () => {
+      await drain(ipfs.pin.add(fixtures.directory.cid))
+      await expectPinned(ipfs, fixtures.directory.cid, pinTypes.recursive)
+
+      const pinChecks = Object.values(fixtures.directory.files).map(file => expectPinned(ipfs, file.cid))
+      return Promise.all(pinChecks)
+    })
+
+    it('should add directly', async () => {
+      await drain(ipfs.pin.add({
+        cid: fixtures.directory.cid,
+        recursive: false
+      }))
+      await Promise.all([
+        expectPinned(ipfs, fixtures.directory.cid, pinTypes.direct),
+        expectPinned(ipfs, fixtures.directory.files[0].cid, false)
+      ])
+    })
+
+    it('should recursively pin parent of direct pin', async () => {
+      await drain(ipfs.pin.add({
+        cid: fixtures.directory.files[0].cid,
+        recursive: false
+      }))
+      await drain(ipfs.pin.add(fixtures.directory.cid))
+      await Promise.all([
+        // file is pinned both directly and indirectly o.O
+        expectPinned(ipfs, fixtures.directory.files[0].cid, pinTypes.direct),
+        expectPinned(ipfs, fixtures.directory.files[0].cid, pinTypes.indirect)
+      ])
+    })
+
+    it('should fail to directly pin a recursive pin', async () => {
+      await drain(ipfs.pin.add(fixtures.directory.cid))
+      return expect(drain(ipfs.pin.add({
+        cid: fixtures.directory.cid,
+        recursive: false
+      })))
+        .to.eventually.be.rejected()
+        .with(/already pinned recursively/)
+    })
+
+    it('should fail to pin a hash not in datastore', function () {
+      this.slow(3 * 1000)
+      this.timeout(5 * 1000)
+      const falseHash = `${`${fixtures.directory.cid}`.slice(0, -2)}ss`
+      return expect(drain(ipfs.pin.add(falseHash, { timeout: '2s' })))
+        .to.eventually.be.rejected()
+        // TODO: http api TimeoutErrors do not have this property
+        // .with.a.property('code').that.equals('ERR_TIMEOUT')
+    })
+
+    it('needs all children in datastore to pin recursively', async function () {
+      this.slow(3 * 1000)
+      this.timeout(5 * 1000)
+      await all(ipfs.block.rm(fixtures.directory.files[0].cid))
+
+      await expect(drain(ipfs.pin.add(fixtures.directory.cid, { timeout: '2s' })))
+        .to.eventually.be.rejected()
+    })
+
+    it('should pin dag-cbor', async () => {
+      const cid = await ipfs.dag.put({}, {
+        format: 'dag-cbor',
+        hashAlg: 'sha2-256'
+      })
+
+      await drain(ipfs.pin.add(cid))
+
+      const pins = await all(ipfs.pin.ls())
+
+      expect(pins).to.deep.include({
+        type: 'recursive',
+        cid
+      })
+    })
+
+    it('should pin raw', async () => {
+      const cid = await ipfs.dag.put(Buffer.alloc(0), {
+        format: 'raw',
+        hashAlg: 'sha2-256'
+      })
+
+      await drain(ipfs.pin.add(cid))
+
+      const pins = await all(ipfs.pin.ls())
+
+      expect(pins).to.deep.include({
+        type: 'recursive',
+        cid
+      })
+    })
+
+    it('should pin dag-cbor with dag-pb child', async () => {
+      const child = await ipfs.dag.put(new DAGNode(Buffer.from(`${Math.random()}`)), {
+        format: 'dag-pb',
+        hashAlg: 'sha2-256'
+      })
+      const parent = await ipfs.dag.put({
+        child
+      }, {
+        format: 'dag-cbor',
+        hashAlg: 'sha2-256'
+      })
+
+      await drain(ipfs.pin.add(parent, {
+        recursive: true
+      }))
+
+      const pins = await all(ipfs.pin.ls())
+
+      expect(pins).to.deep.include({
+        cid: parent,
+        type: 'recursive'
+      })
+      expect(pins).to.deep.include({
+        cid: child,
+        type: 'indirect'
+      })
     })
   })
 }

packages/interface-ipfs-core/src/pin/index.js

@@ -2,9 +2,9 @@
 const { createSuite } = require('../utils/suite')
 
 const tests = {
+  add: require('./add'),
   ls: require('./ls'),
-  rm: require('./rm'),
-  add: require('./add')
+  rm: require('./rm')
 }
 
 module.exports = createSuite(tests)