README.md

@@ -1,6 +1,6 @@
 # Kredits IPFS Pinner
 
-This tool pins the IPFS hashes of a Kredits organisation on an IPFS node.
+This tool pins the IPFS documents of a Kredits organization on any IPFS node.
 
 ## Usage
 
@@ -9,11 +9,11 @@ config flags, in case it is not running on localhost with default ports.)
 
 ### With NPX magic
 
-    npx @kosmos/kredits-ipfs-pinner
+    npx @kredits/ipfs-pinner
 
 ### Global install
 
-    npm install -g @kosmos/kredits-ipfs-pinner
+    npm install -g @kredits/ipfs-pinner
     kredits-ipfs-pinner
 
 ### From repo

config/peers.json

@@ -0,0 +1,6 @@
+[
+  {
+    "ID": "QmRw21aC3TroRVdZhrE2Qh4W6PBA67kbE8p3fNfsVcfW8D",
+    "Addrs": ["/dns4/draco.kosmos.org/tcp/4001"]
+  }
+]

index.js

@@ -1,31 +1,33 @@
 #!/usr/bin/env node
 
-const Kredits = require('kredits-contracts');
+const fs = require('fs');
-const IpfsPinner = require('./lib/ipfs-pinner');
 const debug = require('debug')('ipfs-pinner');
+const Kredits = require('@kredits/contracts');
+const IpfsPinner = require('./lib/ipfs-pinner');
+const defaultPeers = JSON.parse(fs.readFileSync('./config/peers.json'));
 
 const argv = require('yargs')
   .default({
-    network: 'rinkeby',
+    rpcUrl: 'https://rsk-testnet.kosmos.org',
-    apm: 'open.aragonpm.eth',
     host: 'localhost',
     port: '5001',
     protocol: 'http',
-    monitor: true,
+    watch: false,
-    bootstrapNode: '/dns4/barnard.kosmos.org/tcp/4001/ipfs/QmbqZCZ2RzVr4r1UEdFak6ra76kHxtGmfV9r3e1Ev6Tj5D'
+    progress: false,
+    bootstrapNode: `${defaultPeers[0].Addrs[0]}/ipfs/${defaultPeers[0].ID}`
   })
-  .boolean('monitor')
+  .boolean('watch')
+  .boolean('progress')
   .describe({
-    network: 'Ethereum network to connect to',
+    rpcUrl: 'Web3/EVM node RPC URL; alternative to --network',
-    rpcUrl: 'Ethereum node RPC URL; alternative to --network',
-    daoAddress: 'Optional Kredits DAO address',
     host: 'IPFS API host',
     port: 'IPFS API port',
     protocol: 'IPFS API protocol',
-    monitor: 'Monitor contract events for new IPFS documents',
+    watch: 'Monitor contract events for new IPFS documents',
+    progress: 'Show progress bars',
     bootstrapNode: 'IPFS node address to connect to before fetching documents'
   })
-  .example('$0 --network rinkeby --host localhost', 'Pins all existing IPFS documents to the IPFS API running on localhost and monitors for new events.')
+  .example('$0 --host localhost', 'Pins all existing IPFS documents to the IPFS API running on localhost and monitors for new events')
   .argv;
 
 const ipfsConfig = {
@@ -34,38 +36,48 @@ const ipfsConfig = {
   protocol: argv.protocol
 };
 
-console.log(`Using IPFS:`, ipfsConfig);
+debug(`IPFS node:`, ipfsConfig);
 
 (async () => {
-  try {
+  const kredits = await Kredits.for(
-    const kredits = await Kredits.for(
+    { rpcUrl: argv.rpcUrl },
-      { network: argv.network, rpcUrl: argv.rpcUrl },
+    { ipfsConfig: ipfsConfig }
-      { apm: argv.apm, ipfsConfig: ipfsConfig, addresses: { Kernel: argv.daoAddress } }
+  ).init().catch(e => {
-    ).init();
+    console.log('Failed to initialize Kredits:');
+    console.log(e.message);
+    process.exit(1);
+  });
 
-    // check the connection to the IPFS client
+  // TODO redesign IPFS wrapper API and do not use an internal attribute
-    // TODO redesign IPFS wrapper API and do not use an internal attribute
+  const ipfsApi = kredits.ipfs._ipfsAPI;
-    const ipfsApi = kredits.ipfs._ipfsAPI;
 
-    await ipfsApi.id();
+  await ipfsApi.id().catch(e => {
+    console.log('Failed to initialize IPFS:');
+    console.log(e.message);
+    process.exit(1);
+  });
 
-    debug(`Connecting to known IPFS node ${argv.bootstrapNode}`);
+  debug(`Connecting to known IPFS node ${argv.bootstrapNode}`);
-    await ipfsApi.swarm.connect(argv.bootstrapNode);
+  await ipfsApi.swarm.connect(argv.bootstrapNode);
 
-    const ipfsPinner = new IpfsPinner(kredits);
+  const ipfsPinner = new IpfsPinner(kredits, {
+    progress: argv.progress
+  });
 
-    ipfsPinner.pinAll().then(pins => {
+  await ipfsPinner.pinAll().then(cids => {
-      console.log(`Pinned ${pins.length} existing documents`);
+    console.log(`\nSuccessfully pinned ${cids.length} documents`)
-    });
+  });
 
-    ipfsPinner.monitor(pin => {
+  if (argv.watch) {
+    console.log('\nWatching contract events for new documents...');
+
+    ipfsPinner.watch(pin => {
       console.log('Pinned a new document:', pin[0]["hash"]);
     });
-
+  } else {
-    console.log(`Subscribed to DAO: ${kredits.Kernel.contract.address}`);
+    process.exit(0);
-  } catch(e) {
-    console.log('Failed to start');
-    console.log(e);
-    process.exit(1);
   }
+
+  // TODO Add new deployment/DAO/org ID or all contract proxy addresses
+  // console.log(`Subscribed to DAO: ${kredits.Kernel.contract.address}`);
 })();

lib/ipfs-pinner.js

@@ -1,21 +1,62 @@
 const debug = require('debug')('ipfs-pinner');
+const cliProgress = require('cli-progress');
+
+function sleep(ms) {
+  return new Promise(resolve => setTimeout(resolve, ms));
+}
 
 class IpfsPinner {
-  constructor (kredits, ipfsApi) {
+  constructor (kredits, options={}) {
     this.kredits = kredits;
-    this.ipfsApi = ipfsApi || this.kredits.ipfs;
+    this.ipfsApi = this.kredits.ipfs;
+    this.progressBars = !!options.progress && !process.env.DEBUG;
+
+    if (this.progressBars) {
+      this.multibar = new cliProgress.MultiBar({
+        stopOnComplete: true,
+        clearOnComplete: false,
+        hideCursor: false,
+        etaBuffer: 30,
+        format: '{entity} [{bar}] {percentage}% | ETA: {eta_formatted} | {value}/{total}'
+      }, cliProgress.Presets.shades_grey);
+    }
   }
 
   async pinAll () {
-    const contributorHashes  = await this._pinAllFromContract(this.kredits.Contributor);
+    console.log('Pinning IPFS documents for all known items...\n')
-    const contributionHashes = await this._pinAllFromContract(this.kredits.Contribution);
+    const cids = [];
-    const proposalHashes     = await this._pinAllFromContract(this.kredits.Proposal);
+    const promises = [];
+    const contracts = [
+      this.kredits.Contributor,
+      this.kredits.Contribution,
+      // TODO uncomment once we have data here
+      // this.kredits.Reimbursement
+    ]
 
-    return contributorHashes.concat(contributionHashes)
+    for (const contract of contracts) {
-                            .concat(proposalHashes);
+      debug(`Pinning data from ${contract.constructor.name}...`);
+      const itemCount = await contract.count;
+      debug(`${contract.constructor.name} item count:`, itemCount);
+      let bar;
+
+      if (this.progressBars) {
+        bar = this.multibar.create(itemCount, 0);
+        bar.update(0, {entity: `${contract.constructor.name}s`.padEnd(14)});
+      }
+
+      promises.push(this._pinAllFromContract(contract, itemCount, bar)
+                        .then(res => { cids.push(...res); }));
+    }
+
+    await Promise.all(promises);
+
+    // Avoid console output race condition with progress bars finishing update
+    if (this.progressBars) await sleep(1000);
+
+    return cids;
   }
 
-  monitor (callback) {
+  watch (callback) {
     this.kredits.Contribution.on('ContributionAdded', (id) => {
       this.kredits.Contribution.getData(id)
         .then(data => { return this.ipfsApi.pin(data); })
@@ -33,19 +74,43 @@ class IpfsPinner {
     });
   }
 
-  _pinAllFromContract (contract) {
+  async _pinAllFromContract (contract, itemCount, progressBar) {
-    debug(`Pinning data from ${contract.constructor.name}...`);
+    const ipfsApi = this.ipfsApi;
-    return contract.count.then(count => {
+    const progressBars = this.progressBars;
-      let promises = [...Array(count).keys()].map(i => {
+    const ids = [...Array(itemCount).keys()].map(i => i+1);
-        let id = i + 1; // 0 => 1 - ids start with 1 and not with 0
+    const cids = [];
-        debug(`Loading ${contract.constructor.name} #${id}`);
+    const batchSize = 20;
-        return contract.getData(id).then(data => {
+    let position = 0;
-          debug(`Pinning ${contract.constructor.name} #${id}`);
+
-          return this.ipfsApi.pin(data);
+    async function loadAndPin(id) {
-        });
+      let cid;
-      });
+
-      return Promise.all(promises);
+      try {
-    });
+        const data = await contract.getData(id);
+        debug(`Loaded ${contract.constructor.name} #${id}`);
+        cid = await ipfsApi.pin(data);
+        debug(`Pinned ${contract.constructor.name} #${id} at ${cid}`);
+      } catch(e) {
+        debug(`Error while trying to load an pin ${contract.constructor.name} #${id}:`)
+        debug(e);
+        debug(`\nTrying again...`);
+        loadAndPin(id);
+      } finally {
+        cids.push(cid);
+        if (progressBars) { progressBar.increment(); }
+      }
+    }
+
+    while (position < itemCount) {
+      const batchIds = ids.slice(position, position + batchSize);
+
+      await Promise.all(batchIds.map(async id => loadAndPin(id)));
+
+      position += batchSize;
+    }
+
+    return cids;
   }
 }
+
 module.exports = IpfsPinner;

package.json

@@ -20,9 +20,10 @@
   ],
   "license": "MIT",
   "dependencies": {
-    "debug": "^4.1.1",
+    "@kredits/contracts": "7.0.0",
-    "kredits-contracts": "^5.4.0",
+    "cli-progress": "^3.11.2",
-    "yargs": "^13.2.4"
+    "debug": "^4.3.4",
+    "yargs": "^17.6.0"
   },
   "repository": {
     "type": "git",