Pin reimbursement data

Also process loading and pinning functions in batches, to prevent stack
size errors on RSK nodes.

README.md

@@ -1,6 +1,6 @@
 # Kredits IPFS Pinner
 
-This tool pins the IPFS hashes of a Kredits organisation on an IPFS node.
+This tool pins the IPFS documents of a Kredits organization on any IPFS node.
 
 ## Usage
 
@@ -9,11 +9,11 @@ config flags, in case it is not running on localhost with default ports.)
 
 ### With NPX magic
 
-    npx @kosmos/kredits-ipfs-pinner
+    npx @kredits/ipfs-pinner
 
 ### Global install
 
-    npm install -g @kosmos/kredits-ipfs-pinner
+    npm install -g @kredits/ipfs-pinner
     kredits-ipfs-pinner
 
 ### From repo

config/peers.json

@@ -0,0 +1,6 @@
+[
+  {
+    "ID": "QmRw21aC3TroRVdZhrE2Qh4W6PBA67kbE8p3fNfsVcfW8D",
+    "Addrs": ["/dns4/draco.kosmos.org/tcp/4001"]
+  }
+]

index.js

@@ -1,38 +1,30 @@
 #!/usr/bin/env node
 
-const Kredits = require('kredits-contracts');
+const fs = require('fs');
-const IpfsPinner = require('./lib/ipfs-pinner');
 const debug = require('debug')('ipfs-pinner');
-const { inspect } = require('util');
+const Kredits = require('@kredits/contracts');
-
+const IpfsPinner = require('./lib/ipfs-pinner');
-// TODO make configurable
+const defaultPeers = JSON.parse(fs.readFileSync('./config/peers.json'));
-const peerId = "QmbqZCZ2RzVr4r1UEdFak6ra76kHxtGmfV9r3e1Ev6Tj5D";
-const peerAddrs = [ "/dns4/barnard.kosmos.org/tcp/4001" ];
-const bootstrapNode = `${peerAddrs[0]}/ipfs/${peerId}`;
-const peerConfig = { "ID": peerId, "Addrs": [ peerAddrs ] };
 
 const argv = require('yargs')
   .default({
-    network: 'rinkeby',
+    rpcUrl: 'https://rsk-testnet.kosmos.org',
-    apm: 'open.aragonpm.eth',
     host: 'localhost',
     port: '5001',
     protocol: 'http',
     monitor: true,
-    bootstrapNode: bootstrapNode
+    bootstrapNode: `${defaultPeers[0].Addrs[0]}/ipfs/${defaultPeers[0].ID}`
   })
   .boolean('monitor')
   .describe({
-    network: 'Ethereum network to connect to',
+    rpcUrl: 'Web3/EVM node RPC URL; alternative to --network',
-    rpcUrl: 'Ethereum node RPC URL; alternative to --network',
-    daoAddress: 'Optional Kredits DAO address',
     host: 'IPFS API host',
     port: 'IPFS API port',
     protocol: 'IPFS API protocol',
     monitor: 'Monitor contract events for new IPFS documents',
     bootstrapNode: 'IPFS node address to connect to before fetching documents'
   })
-  .example('$0 --network rinkeby --host localhost', 'Pins all existing IPFS documents to the IPFS API running on localhost and monitors for new events.')
+  .example('$0 --host localhost', 'Pins all existing IPFS documents to the IPFS API running on localhost and monitors for new events')
   .argv;
 
 const ipfsConfig = {
@@ -46,45 +38,31 @@ console.log(`Using IPFS:`, ipfsConfig);
 (async () => {
   try {
     const kredits = await Kredits.for(
-      { network: argv.network, rpcUrl: argv.rpcUrl },
+      { rpcUrl: argv.rpcUrl },
-      { apm: argv.apm, ipfsConfig: ipfsConfig, addresses: { Kernel: argv.daoAddress } }
+      { ipfsConfig: ipfsConfig }
     ).init();
 
-    // check the connection to the IPFS client
+    // Check the connection to the IPFS client
     // TODO redesign IPFS wrapper API and do not use an internal attribute
     const ipfsApi = kredits.ipfs._ipfsAPI;
 
     await ipfsApi.id();
 
-    // debug(`Connecting to known IPFS node ${argv.bootstrapNode}`);
+    debug(`Connecting to known IPFS node ${argv.bootstrapNode}`);
-    // await ipfsApi.swarm.connect(argv.bootstrapNode);
+    await ipfsApi.swarm.connect(argv.bootstrapNode);
 
-    let peers;
+    const ipfsPinner = new IpfsPinner(kredits);
 
-    try {
+    ipfsPinner.pinAll().then(pins => {
-      peers = await ipfsApi.config.get('Peering.Peers');
+      console.log(`Pinned ${pins.length} existing documents`);
-    } catch(e) { /* No peers configured */ }
+    });
 
-    if (peers) {
+    ipfsPinner.monitor(pin => {
-      // TODO check for kosmos peer
+      console.log('Pinned a new document:', pin[0]["hash"]);
-      debug('IPFS peers configured:');
+    });
-      debug(inspect(peers.map(p => p.ID)));
-    } else {
-      debug(`Configuring persistent IPFS peer: ${peerId}`);
-      await ipfsApi.config.set('Peering', { "Peers": [ peerConfig ]});
-    }
 
-    // const ipfsPinner = new IpfsPinner(kredits);
+    // TODO Add new deployment/DAO/org ID or all contract proxy addresses
-
+    // console.log(`Subscribed to DAO: ${kredits.Kernel.contract.address}`);
-    // ipfsPinner.pinAll().then(pins => {
-    //   console.log(`Pinned ${pins.length} existing documents`);
-    // });
-    //
-    // ipfsPinner.monitor(pin => {
-    //   console.log('Pinned a new document:', pin[0]["hash"]);
-    // });
-
-    console.log(`Subscribed to DAO: ${kredits.Kernel.contract.address}`);
   } catch(e) {
     console.log('Failed to start');
     console.log(e);

lib/ipfs-pinner.js

@@ -1,5 +1,16 @@
 const debug = require('debug')('ipfs-pinner');
 
+async function promiseAllInBatches(task, items, batchSize) {
+  let position = 0;
+  let results = [];
+  while (position < items.length) {
+    const itemsForBatch = items.slice(position, position + batchSize);
+    results = [...results, ...await Promise.allSettled(itemsForBatch.map(item => task(item)))];
+    position += batchSize;
+  }
+  return results;
+}
+
 class IpfsPinner {
   constructor (kredits, ipfsApi) {
     this.kredits = kredits;
@@ -7,12 +18,12 @@ class IpfsPinner {
   }
 
   async pinAll () {
-    const contributorHashes  = await this._pinAllFromContract(this.kredits.Contributor);
+    const contributorHashes   = await this._pinAllFromContract(this.kredits.Contributor);
-    const contributionHashes = await this._pinAllFromContract(this.kredits.Contribution);
+    const contributionHashes  = await this._pinAllFromContract(this.kredits.Contribution);
-    const proposalHashes     = await this._pinAllFromContract(this.kredits.Proposal);
+    const reimbursementHashes = await this._pinAllFromContract(this.kredits.Reimbursement);
 
     return contributorHashes.concat(contributionHashes)
-                            .concat(proposalHashes);
+                            .concat(reimbursementHashes);
   }
 
   monitor (callback) {
@@ -33,19 +44,25 @@ class IpfsPinner {
     });
   }
 
-  _pinAllFromContract (contract) {
+  async _pinAllFromContract (contract) {
     debug(`Pinning data from ${contract.constructor.name}...`);
-    return contract.count.then(count => {
+    const count = await contract.count;
-      let promises = [...Array(count).keys()].map(i => {
+    debug('Item count:', count);
-        let id = i + 1; // 0 => 1 - ids start with 1 and not with 0
+    const ids = [...Array(count).keys()].map(i => i+1);
-        debug(`Loading ${contract.constructor.name} #${id}`);
+    const cids = [];
-        return contract.getData(id).then(data => {
+
-          debug(`Pinning ${contract.constructor.name} #${id}`);
+    async function loadAndPin (id) {
-          return this.ipfsApi.pin(data);
+     debug(`Loading ${contract.constructor.name} #${id}`);
-        });
+      return contract.getData(id).then(data => {
+        debug(`Pinning ${contract.constructor.name} #${id}`);
+        return this.ipfsApi.pin(data).then(cid => cids.push(cid));
       });
-      return Promise.all(promises);
+    }
-    });
+
+    await promiseAllInBatches(loadAndPin.bind(this), ids, 100);
+
+    return cids;
   }
 }
+
 module.exports = IpfsPinner;

package.json

@@ -20,9 +20,9 @@
   ],
   "license": "MIT",
   "dependencies": {
-    "debug": "^4.1.1",
+    "@kredits/contracts": "git+https://gitea.kosmos.org/kredits/contracts#6e0ec87",
-    "kredits-contracts": "^5.4.0",
+    "debug": "^4.3.4",
-    "yargs": "^13.2.4"
+    "yargs": "^17.6.0"
   },
   "repository": {
     "type": "git",