Pin reimbursement data

Also process loading and pinning functions in batches, to prevent stack
size errors on RSK nodes.

README.md

@@ -1,6 +1,6 @@
 # Kredits IPFS Pinner
 
-This tool pins the IPFS hashes of a Kredits organisation on an IPFS node.
+This tool pins the IPFS documents of a Kredits organization on any IPFS node.
 
 ## Usage
 
@@ -9,11 +9,11 @@ config flags, in case it is not running on localhost with default ports.)
 
 ### With NPX magic
 
-    npx @kosmos/kredits-ipfs-pinner
+    npx @kredits/ipfs-pinner
 
 ### Global install
 
-    npm install -g @kosmos/kredits-ipfs-pinner
+    npm install -g @kredits/ipfs-pinner
     kredits-ipfs-pinner
 
 ### From repo

config/peers.json

@@ -0,0 +1,6 @@
+[
+  {
+    "ID": "QmRw21aC3TroRVdZhrE2Qh4W6PBA67kbE8p3fNfsVcfW8D",
+    "Addrs": ["/dns4/draco.kosmos.org/tcp/4001"]
+  }
+]

index.js

@@ -1,31 +1,30 @@
 #!/usr/bin/env node
 
-const Kredits = require('kredits-contracts');
+const fs = require('fs');
-const IpfsPinner = require('./lib/ipfs-pinner');
 const debug = require('debug')('ipfs-pinner');
+const Kredits = require('@kredits/contracts');
+const IpfsPinner = require('./lib/ipfs-pinner');
+const defaultPeers = JSON.parse(fs.readFileSync('./config/peers.json'));
 
 const argv = require('yargs')
   .default({
-    network: 'rinkeby',
+    rpcUrl: 'https://rsk-testnet.kosmos.org',
-    apm: 'open.aragonpm.eth',
     host: 'localhost',
     port: '5001',
     protocol: 'http',
     monitor: true,
-    bootstrapNode: '/ip4/46.4.18.160/tcp/4001/ipfs/QmZ4Lpzhz8bafbTYvEMMCmrbrMM4JfyHDR23WbCSAd9bo7'
+    bootstrapNode: `${defaultPeers[0].Addrs[0]}/ipfs/${defaultPeers[0].ID}`
   })
   .boolean('monitor')
   .describe({
-    network: 'Ethereum network to connect to',
+    rpcUrl: 'Web3/EVM node RPC URL; alternative to --network',
-    rpcUrl: 'Ethereum node RPC URL; alternative to --network',
-    daoAddress: 'Optional Kredits DAO address',
     host: 'IPFS API host',
     port: 'IPFS API port',
     protocol: 'IPFS API protocol',
     monitor: 'Monitor contract events for new IPFS documents',
     bootstrapNode: 'IPFS node address to connect to before fetching documents'
   })
-  .example('$0 --network rinkeby --host localhost', 'Pins all existing IPFS documents to the IPFS API running on localhost and monitors for new events.')
+  .example('$0 --host localhost', 'Pins all existing IPFS documents to the IPFS API running on localhost and monitors for new events')
   .argv;
 
 const ipfsConfig = {
@@ -39,11 +38,11 @@ console.log(`Using IPFS:`, ipfsConfig);
 (async () => {
   try {
     const kredits = await Kredits.for(
-      { network: argv.network, rpcUrl: argv.rpcUrl },
+      { rpcUrl: argv.rpcUrl },
-      { apm: argv.apm, ipfsConfig: ipfsConfig, addresses: { Kernel: argv.daoAddress } }
+      { ipfsConfig: ipfsConfig }
     ).init();
 
-    // check the connection to the IPFS client
+    // Check the connection to the IPFS client
     // TODO redesign IPFS wrapper API and do not use an internal attribute
     const ipfsApi = kredits.ipfs._ipfsAPI;
 
@@ -62,7 +61,8 @@ console.log(`Using IPFS:`, ipfsConfig);
       console.log('Pinned a new document:', pin[0]["hash"]);
     });
 
-    console.log(`Subscribed to DAO: ${kredits.Kernel.contract.address}`);
+    // TODO Add new deployment/DAO/org ID or all contract proxy addresses
+    // console.log(`Subscribed to DAO: ${kredits.Kernel.contract.address}`);
   } catch(e) {
     console.log('Failed to start');
     console.log(e);

lib/ipfs-pinner.js

@@ -1,5 +1,16 @@
 const debug = require('debug')('ipfs-pinner');
 
+async function promiseAllInBatches(task, items, batchSize) {
+  let position = 0;
+  let results = [];
+  while (position < items.length) {
+    const itemsForBatch = items.slice(position, position + batchSize);
+    results = [...results, ...await Promise.allSettled(itemsForBatch.map(item => task(item)))];
+    position += batchSize;
+  }
+  return results;
+}
+
 class IpfsPinner {
   constructor (kredits, ipfsApi) {
     this.kredits = kredits;
@@ -9,10 +20,10 @@ class IpfsPinner {
   async pinAll () {
     const contributorHashes   = await this._pinAllFromContract(this.kredits.Contributor);
     const contributionHashes  = await this._pinAllFromContract(this.kredits.Contribution);
-    const proposalHashes     = await this._pinAllFromContract(this.kredits.Proposal);
+    const reimbursementHashes = await this._pinAllFromContract(this.kredits.Reimbursement);
 
     return contributorHashes.concat(contributionHashes)
-                            .concat(proposalHashes);
+                            .concat(reimbursementHashes);
   }
 
   monitor (callback) {
@@ -33,19 +44,25 @@ class IpfsPinner {
     });
   }
 
-  _pinAllFromContract (contract) {
+  async _pinAllFromContract (contract) {
     debug(`Pinning data from ${contract.constructor.name}...`);
-    return contract.count.then(count => {
+    const count = await contract.count;
-      let promises = [...Array(count).keys()].map(i => {
+    debug('Item count:', count);
-        let id = i + 1; // 0 => 1 - ids start with 1 and not with 0
+    const ids = [...Array(count).keys()].map(i => i+1);
+    const cids = [];
+
+    async function loadAndPin (id) {
      debug(`Loading ${contract.constructor.name} #${id}`);
       return contract.getData(id).then(data => {
         debug(`Pinning ${contract.constructor.name} #${id}`);
-          return this.ipfsApi.pin(data);
+        return this.ipfsApi.pin(data).then(cid => cids.push(cid));
-        });
-      });
-      return Promise.all(promises);
       });
     }
+
+    await promiseAllInBatches(loadAndPin.bind(this), ids, 100);
+
+    return cids;
   }
+}
+
 module.exports = IpfsPinner;

package.json

@@ -20,9 +20,9 @@
   ],
   "license": "MIT",
   "dependencies": {
-    "debug": "^4.1.1",
+    "@kredits/contracts": "git+https://gitea.kosmos.org/kredits/contracts#6e0ec87",
-    "kredits-contracts": "^5.4.0",
+    "debug": "^4.3.4",
-    "yargs": "^13.2.4"
+    "yargs": "^17.6.0"
   },
   "repository": {
     "type": "git",