Pin reimbursement data

Also process loading and pinning functions in batches, to prevent stack
size errors on RSK nodes.

README.md

@@ -1,6 +1,6 @@
 # Kredits IPFS Pinner
 
-This tool pins the IPFS hashes of a Kredits organisation on an IPFS node.
+This tool pins the IPFS documents of a Kredits organization on any IPFS node.
 
 ## Usage
 
@@ -9,11 +9,11 @@ config flags, in case it is not running on localhost with default ports.)
 
 ### With NPX magic
 
-    npx @kosmos/kredits-ipfs-pinner
+    npx @kredits/ipfs-pinner
 
 ### Global install
 
-    npm install -g @kosmos/kredits-ipfs-pinner
+    npm install -g @kredits/ipfs-pinner
     kredits-ipfs-pinner
 
 ### From repo

config/peers.json

@@ -0,0 +1,6 @@
+[
+  {
+    "ID": "QmRw21aC3TroRVdZhrE2Qh4W6PBA67kbE8p3fNfsVcfW8D",
+    "Addrs": ["/dns4/draco.kosmos.org/tcp/4001"]
+  }
+]

index.js

@@ -1,21 +1,30 @@
 #!/usr/bin/env node
 
-const Kredits = require('kredits-contracts');
+const fs = require('fs');
+const debug = require('debug')('ipfs-pinner');
+const Kredits = require('@kredits/contracts');
 const IpfsPinner = require('./lib/ipfs-pinner');
+const defaultPeers = JSON.parse(fs.readFileSync('./config/peers.json'));
 
 const argv = require('yargs')
-  .default({ network: 'rinkeby', apm: 'open.aragonpm.eth', host: 'localhost', port: '5001', protocol: 'http', monitor: true })
+  .default({
+    rpcUrl: 'https://rsk-testnet.kosmos.org',
+    host: 'localhost',
+    port: '5001',
+    protocol: 'http',
+    monitor: true,
+    bootstrapNode: `${defaultPeers[0].Addrs[0]}/ipfs/${defaultPeers[0].ID}`
+  })
   .boolean('monitor')
   .describe({
-    network: 'Ethereum network to connect to',
+    rpcUrl: 'Web3/EVM node RPC URL; alternative to --network',
-    rpcUrl: 'Ethereum node RPC URL; alternative to --network',
-    daoAddress: 'Optional Kredits DAO address',
     host: 'IPFS API host',
     port: 'IPFS API port',
     protocol: 'IPFS API protocol',
-    monitor: 'Monitor contract events for new IPFS documents'
+    monitor: 'Monitor contract events for new IPFS documents',
+    bootstrapNode: 'IPFS node address to connect to before fetching documents'
   })
-  .example('$0 --network rinkeby --host localhost', 'Pins all existing IPFS documents to the IPFS API running on localhost and monitors for new events.')
+  .example('$0 --host localhost', 'Pins all existing IPFS documents to the IPFS API running on localhost and monitors for new events')
   .argv;
 
 const ipfsConfig = {
@@ -29,13 +38,18 @@ console.log(`Using IPFS:`, ipfsConfig);
 (async () => {
   try {
     const kredits = await Kredits.for(
-      { network: argv.network, rpcUrl: argv.rpcUrl },
+      { rpcUrl: argv.rpcUrl },
-      { apm: argv.apm, ipfsConfig: ipfsConfig, addresses: { Kernel: argv.daoAddress } }
+      { ipfsConfig: ipfsConfig }
     ).init();
 
-    // check the connection to the IPFS client
+    // Check the connection to the IPFS client
     // TODO redesign IPFS wrapper API and do not use an internal attribute
-    await kredits.ipfs._ipfsAPI.id();
+    const ipfsApi = kredits.ipfs._ipfsAPI;
+
+    await ipfsApi.id();
+
+    debug(`Connecting to known IPFS node ${argv.bootstrapNode}`);
+    await ipfsApi.swarm.connect(argv.bootstrapNode);
 
     const ipfsPinner = new IpfsPinner(kredits);
 
@@ -47,7 +61,8 @@ console.log(`Using IPFS:`, ipfsConfig);
       console.log('Pinned a new document:', pin[0]["hash"]);
     });
 
-    console.log(`Subscribed to DAO: ${kredits.Kernel.contract.address}`);
+    // TODO Add new deployment/DAO/org ID or all contract proxy addresses
+    // console.log(`Subscribed to DAO: ${kredits.Kernel.contract.address}`);
   } catch(e) {
     console.log('Failed to start');
     console.log(e);

lib/ipfs-pinner.js

@@ -1,5 +1,16 @@
 const debug = require('debug')('ipfs-pinner');
 
+async function promiseAllInBatches(task, items, batchSize) {
+  let position = 0;
+  let results = [];
+  while (position < items.length) {
+    const itemsForBatch = items.slice(position, position + batchSize);
+    results = [...results, ...await Promise.allSettled(itemsForBatch.map(item => task(item)))];
+    position += batchSize;
+  }
+  return results;
+}
+
 class IpfsPinner {
   constructor (kredits, ipfsApi) {
     this.kredits = kredits;
@@ -9,10 +20,10 @@ class IpfsPinner {
   async pinAll () {
     const contributorHashes   = await this._pinAllFromContract(this.kredits.Contributor);
     const contributionHashes  = await this._pinAllFromContract(this.kredits.Contribution);
-    const proposalHashes     = await this._pinAllFromContract(this.kredits.Proposal);
+    const reimbursementHashes = await this._pinAllFromContract(this.kredits.Reimbursement);
 
     return contributorHashes.concat(contributionHashes)
-                            .concat(proposalHashes);
+                            .concat(reimbursementHashes);
   }
 
   monitor (callback) {
@@ -33,19 +44,25 @@ class IpfsPinner {
     });
   }
 
-  _pinAllFromContract (contract) {
+  async _pinAllFromContract (contract) {
     debug(`Pinning data from ${contract.constructor.name}...`);
-    return contract.count.then(count => {
+    const count = await contract.count;
-      let promises = [...Array(count).keys()].map(i => {
+    debug('Item count:', count);
-        let id = i + 1; // 0 => 1 - ids start with 1 and not with 0
+    const ids = [...Array(count).keys()].map(i => i+1);
+    const cids = [];
+
+    async function loadAndPin (id) {
      debug(`Loading ${contract.constructor.name} #${id}`);
       return contract.getData(id).then(data => {
         debug(`Pinning ${contract.constructor.name} #${id}`);
-          return this.ipfsApi.pin(data);
+        return this.ipfsApi.pin(data).then(cid => cids.push(cid));
-        });
-      });
-      return Promise.all(promises);
       });
     }
+
+    await promiseAllInBatches(loadAndPin.bind(this), ids, 100);
+
+    return cids;
+  }
 }
+
 module.exports = IpfsPinner;

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@kosmos/kredits-ipfs-pinner",
-  "version": "1.0.3",
+  "version": "1.2.0",
   "description": "Pins IPFS data of a Kredits organization on an IPFS node",
   "main": "index.js",
   "scripts": {
@@ -20,9 +20,9 @@
   ],
   "license": "MIT",
   "dependencies": {
-    "debug": "^4.1.1",
+    "@kredits/contracts": "git+https://gitea.kosmos.org/kredits/contracts#6e0ec87",
-    "kredits-contracts": "^5.4.0",
+    "debug": "^4.3.4",
-    "yargs": "^13.2.4"
+    "yargs": "^17.6.0"
   },
   "repository": {
     "type": "git",