2.3.0

Reviewed-on: #16

.gitea/release-drafter.yml

@@ -0,0 +1,14 @@
+name-template: 'v$RESOLVED_VERSION'
+tag-template: 'v$RESOLVED_VERSION'
+version-resolver:
+  major:
+    labels:
+      - release/major
+  minor:
+    labels:
+      - release/minor
+      - feature
+  patch:
+    labels:
+      - release/patch
+  default: patch

.gitea/workflows/release_drafter.yml

@@ -0,0 +1,11 @@
+name: Release Drafter
+on:
+  pull_request:
+    types: [closed]
+jobs:
+  release_drafter_job:
+    name: Update release notes draft
+    runs-on: ubuntu-latest
+    steps:
+      - name: Release Drafter
+        uses: https://github.com/raucao/gitea-release-drafter@dev

README.md

@@ -1,3 +1,5 @@
+[![npm](https://img.shields.io/npm/v/@kredits/ipfs-pinner.svg)](https://www.npmjs.com/package/@kredits/ipfs-pinner)
+
 # Kredits IPFS Pinner
 
 This tool pins the IPFS documents of a Kredits organization on any IPFS node.
@@ -25,3 +27,21 @@ Clone the Git repository, then use npm scripts from its root directory.
 Or with debug output:
 
     npm run debug
+
+### CLI options
+
+Display progress bars:
+
+    kredits-ipfs-pinner --progress
+
+Or with npm:
+
+    npm start -- --progress
+
+Keep running and watch for new data/documents:
+
+    kredits-ipfs-pinner --watch
+
+Multiple options at once:
+
+    kredits-ipfs-pinner --progress --watch

index.js

@@ -12,16 +12,19 @@ const argv = require('yargs')
     host: 'localhost',
     port: '5001',
     protocol: 'http',
-    monitor: true,
+    watch: false,
+    progress: false,
     bootstrapNode: `${defaultPeers[0].Addrs[0]}/ipfs/${defaultPeers[0].ID}`
   })
-  .boolean('monitor')
+  .boolean('watch')
+  .boolean('progress')
   .describe({
     rpcUrl: 'Web3/EVM node RPC URL; alternative to --network',
     host: 'IPFS API host',
     port: 'IPFS API port',
     protocol: 'IPFS API protocol',
-    monitor: 'Monitor contract events for new IPFS documents',
+    watch: 'Monitor contract events for new IPFS documents',
+    progress: 'Show progress bars',
     bootstrapNode: 'IPFS node address to connect to before fetching documents'
   })
   .example('$0 --host localhost', 'Pins all existing IPFS documents to the IPFS API running on localhost and monitors for new events')
@@ -33,39 +36,55 @@ const ipfsConfig = {
   protocol: argv.protocol
 };
 
-console.log(`Using IPFS:`, ipfsConfig);
+debug(`IPFS node:`, ipfsConfig);
 
 (async () => {
-  try {
-    const kredits = await Kredits.for(
-      { rpcUrl: argv.rpcUrl },
-      { ipfsConfig: ipfsConfig }
-    ).init();
+  const kredits = await Kredits.for(
+    { rpcUrl: argv.rpcUrl },
+    { ipfsConfig: ipfsConfig }
+  ).init().catch(e => {
+    console.log('Failed to initialize Kredits:');
+    console.log(e.message);
+    process.exit(1);
+  });
 
-    // Check the connection to the IPFS client
-    // TODO redesign IPFS wrapper API and do not use an internal attribute
-    const ipfsApi = kredits.ipfs._ipfsAPI;
+  // TODO redesign IPFS wrapper API and do not use an internal attribute
+  const ipfsApi = kredits.ipfs._ipfsAPI;
+  let connectBootstrapNode = true;
 
-    await ipfsApi.id();
+  await ipfsApi.id().then(res => {
+    if (res.id === argv.bootstrapNode.split('/')[argv.bootstrapNode.split('/').length-1]) {
+      connectBootstrapNode = false;
+    }
+  }).catch(e => {
+    console.log('Failed to initialize IPFS:');
+    console.log(e.message);
+    process.exit(1);
+  });
 
+  if (connectBootstrapNode) {
     debug(`Connecting to known IPFS node ${argv.bootstrapNode}`);
     await ipfsApi.swarm.connect(argv.bootstrapNode);
+  }
 
-    const ipfsPinner = new IpfsPinner(kredits);
+  const ipfsPinner = new IpfsPinner(kredits, {
+    progress: argv.progress
+  });
 
-    ipfsPinner.pinAll().then(pins => {
-      console.log(`Pinned ${pins.length} existing documents`);
-    });
+  await ipfsPinner.pinAll().then(cids => {
+    console.log(`\nSuccessfully pinned ${cids.length} documents`)
+  });
 
-    ipfsPinner.monitor(pin => {
+  if (argv.watch) {
+    console.log('\nWatching contract events for new documents...');
+
+    ipfsPinner.watch(pin => {
       console.log('Pinned a new document:', pin[0]["hash"]);
     });
-
-    // TODO Add new deployment/DAO/org ID or all contract proxy addresses
-    // console.log(`Subscribed to DAO: ${kredits.Kernel.contract.address}`);
-  } catch(e) {
-    console.log('Failed to start');
-    console.log(e);
-    process.exit(1);
+  } else {
+    process.exit(0);
   }
+
+  // TODO Add new deployment/DAO/org ID or all contract proxy addresses
+  // console.log(`Subscribed to DAO: ${kredits.Kernel.contract.address}`);
 })();

lib/ipfs-pinner.js

@@ -1,32 +1,61 @@
 const debug = require('debug')('ipfs-pinner');
+const cliProgress = require('cli-progress');
 
-async function promiseAllInBatches(task, items, batchSize) {
-  let position = 0;
-  let results = [];
-  while (position < items.length) {
-    const itemsForBatch = items.slice(position, position + batchSize);
-    results = [...results, ...await Promise.allSettled(itemsForBatch.map(item => task(item)))];
-    position += batchSize;
-  }
-  return results;
+function sleep(ms) {
+  return new Promise(resolve => setTimeout(resolve, ms));
 }
 
 class IpfsPinner {
-  constructor (kredits, ipfsApi) {
+  constructor (kredits, options={}) {
     this.kredits = kredits;
-    this.ipfsApi = ipfsApi || this.kredits.ipfs;
+    this.ipfsApi = this.kredits.ipfs;
+    this.progressBars = !!options.progress && !process.env.DEBUG;
+
+    if (this.progressBars) {
+      this.multibar = new cliProgress.MultiBar({
+        stopOnComplete: true,
+        clearOnComplete: false,
+        hideCursor: false,
+        etaBuffer: 30,
+        format: '{entity} [{bar}] {percentage}% | ETA: {eta_formatted} | {value}/{total}'
+      }, cliProgress.Presets.shades_grey);
+    }
   }
 
   async pinAll () {
-    const contributorHashes   = await this._pinAllFromContract(this.kredits.Contributor);
-    const contributionHashes  = await this._pinAllFromContract(this.kredits.Contribution);
-    const reimbursementHashes = await this._pinAllFromContract(this.kredits.Reimbursement);
+    console.log('Pinning IPFS documents for all known items...\n')
+    const cids = [];
+    const promises = [];
+    const contracts = [
+      this.kredits.Contributor,
+      this.kredits.Contribution,
+      this.kredits.Reimbursement
+    ]
 
-    return contributorHashes.concat(contributionHashes)
-                            .concat(reimbursementHashes);
+    for (const contract of contracts) {
+      debug(`Pinning data from ${contract.constructor.name}...`);
+      const itemCount = await contract.count;
+      debug(`${contract.constructor.name} item count:`, itemCount);
+      let bar;
+
+      if (this.progressBars) {
+        bar = this.multibar.create(itemCount, 0);
+        bar.update(0, {entity: `${contract.constructor.name}s`.padEnd(14)});
+      }
+
+      promises.push(this._pinAllFromContract(contract, itemCount, bar)
+                        .then(res => { cids.push(...res); }));
+    }
+
+    await Promise.all(promises);
+
+    // Avoid console output race condition with progress bars finishing update
+    if (this.progressBars) await sleep(1000);
+
+    return cids;
   }
 
-  monitor (callback) {
+  watch (callback) {
     this.kredits.Contribution.on('ContributionAdded', (id) => {
       this.kredits.Contribution.getData(id)
         .then(data => { return this.ipfsApi.pin(data); })
@@ -42,24 +71,47 @@ class IpfsPinner {
         .then(data => { return this.ipfsApi.pin(data); })
         .then(callback);
     });
+    this.kredits.Reimbursement.on('ReimbursementAdded', (id) => {
+      this.kredits.Reimbursement.getData(id)
+        .then(data => { return this.ipfsApi.pin(data); })
+        .then(callback);
+    });
   }
 
-  async _pinAllFromContract (contract) {
-    debug(`Pinning data from ${contract.constructor.name}...`);
-    const count = await contract.count;
-    debug('Item count:', count);
-    const ids = [...Array(count).keys()].map(i => i+1);
+  async _pinAllFromContract (contract, itemCount, progressBar) {
+    const ipfsApi = this.ipfsApi;
+    const progressBars = this.progressBars;
+    const ids = [...Array(itemCount).keys()].map(i => i+1);
     const cids = [];
+    const batchSize = 20;
+    let position = 0;
 
-    async function loadAndPin (id) {
-     debug(`Loading ${contract.constructor.name} #${id}`);
-      return contract.getData(id).then(data => {
-        debug(`Pinning ${contract.constructor.name} #${id}`);
-        return this.ipfsApi.pin(data).then(cid => cids.push(cid));
-      });
+    async function loadAndPin(id) {
+      let cid;
+
+      try {
+        const data = await contract.getData(id);
+        debug(`Loaded ${contract.constructor.name} #${id}`);
+        cid = await ipfsApi.pin(data);
+        debug(`Pinned ${contract.constructor.name} #${id} at ${cid}`);
+      } catch(e) {
+        debug(`Error while trying to load and pin ${contract.constructor.name} #${id}:`)
+        debug(e);
+        debug(`\nTrying again...`);
+        loadAndPin(id);
+      } finally {
+        cids.push(cid);
+        if (progressBars) { progressBar.increment(); }
+      }
     }
 
-    await promiseAllInBatches(loadAndPin.bind(this), ids, 100);
+    while (position < itemCount) {
+      const batchIds = ids.slice(position, position + batchSize);
+
+      await Promise.all(batchIds.map(async id => loadAndPin(id)));
+
+      position += batchSize;
+    }
 
     return cids;
   }

package-lock.json

@@ -1,15 +1,16 @@
 {
-  "name": "@kosmos/kredits-ipfs-pinner",
-  "version": "1.2.0",
+  "name": "@kredits/ipfs-pinner",
+  "version": "2.3.0",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
-      "name": "@kosmos/kredits-ipfs-pinner",
-      "version": "1.2.0",
+      "name": "@kredits/ipfs-pinner",
+      "version": "2.3.0",
       "license": "MIT",
       "dependencies": {
-        "@kredits/contracts": "git+https://gitea.kosmos.org/kredits/contracts#6e0ec87",
+        "@kredits/contracts": "^7.4.0",
+        "cli-progress": "^3.11.2",
         "debug": "^4.3.4",
         "yargs": "^17.6.0"
       },
@@ -720,9 +721,9 @@
       "integrity": "sha512-yOTK5WiXFDNAitPByMabE365aEEzFHgSUSgAssbJWt7BZ80HQSVu8XWrQiTbFbCkoIBmXwPP/RoxgXJQVgZTFQ=="
     },
     "node_modules/@kredits/contracts": {
-      "version": "7.0.0-beta.0",
-      "resolved": "git+https://gitea.kosmos.org/kredits/contracts#6e0ec8741e61b51fb5c9c636da4e8d3610d090ac",
-      "license": "MIT",
+      "version": "7.4.0",
+      "resolved": "https://registry.npmjs.org/@kredits/contracts/-/contracts-7.4.0.tgz",
+      "integrity": "sha512-J3KGqZTZAcwJwjKziyt9PKOiNhme5e5+vVntusqbvS9BJTq8gFc7wr+l9ZMPEel7dl85KopzfpVesYbQ/+f33Q==",
       "dependencies": {
         "@kosmos/schemas": "^3.1.0",
         "ethers": "^5.4.7",
@@ -930,6 +931,17 @@
         "cborg": "cli.js"
       }
     },
+    "node_modules/cli-progress": {
+      "version": "3.11.2",
+      "resolved": "https://registry.npmjs.org/cli-progress/-/cli-progress-3.11.2.tgz",
+      "integrity": "sha512-lCPoS6ncgX4+rJu5bS3F/iCz17kZ9MPZ6dpuTtI0KXKABkhyXIdYB3Inby1OpaGti3YlI3EeEkM9AuWpelJrVA==",
+      "dependencies": {
+        "string-width": "^4.2.3"
+      },
+      "engines": {
+        "node": ">=4"
+      }
+    },
     "node_modules/cliui": {
       "version": "8.0.1",
       "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz",
@@ -2233,8 +2245,9 @@
       "integrity": "sha512-yOTK5WiXFDNAitPByMabE365aEEzFHgSUSgAssbJWt7BZ80HQSVu8XWrQiTbFbCkoIBmXwPP/RoxgXJQVgZTFQ=="
     },
     "@kredits/contracts": {
-      "version": "git+https://gitea.kosmos.org/kredits/contracts#6e0ec8741e61b51fb5c9c636da4e8d3610d090ac",
-      "from": "@kredits/contracts@git+https://gitea.kosmos.org/kredits/contracts#6e0ec87",
+      "version": "7.4.0",
+      "resolved": "https://registry.npmjs.org/@kredits/contracts/-/contracts-7.4.0.tgz",
+      "integrity": "sha512-J3KGqZTZAcwJwjKziyt9PKOiNhme5e5+vVntusqbvS9BJTq8gFc7wr+l9ZMPEel7dl85KopzfpVesYbQ/+f33Q==",
       "requires": {
         "@kosmos/schemas": "^3.1.0",
         "ethers": "^5.4.7",
@@ -2402,6 +2415,14 @@
       "resolved": "https://registry.npmjs.org/cborg/-/cborg-1.9.5.tgz",
       "integrity": "sha512-fLBv8wmqtlXqy1Yu+pHzevAIkW6k2K0ZtMujNzWphLsA34vzzg9BHn+5GmZqOJkSA9V7EMKsWrf6K976c1QMjQ=="
     },
+    "cli-progress": {
+      "version": "3.11.2",
+      "resolved": "https://registry.npmjs.org/cli-progress/-/cli-progress-3.11.2.tgz",
+      "integrity": "sha512-lCPoS6ncgX4+rJu5bS3F/iCz17kZ9MPZ6dpuTtI0KXKABkhyXIdYB3Inby1OpaGti3YlI3EeEkM9AuWpelJrVA==",
+      "requires": {
+        "string-width": "^4.2.3"
+      }
+    },
     "cliui": {
       "version": "8.0.1",
       "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz",

package.json

@@ -1,6 +1,6 @@
 {
-  "name": "@kosmos/kredits-ipfs-pinner",
-  "version": "1.2.0",
+  "name": "@kredits/ipfs-pinner",
+  "version": "2.3.0",
   "description": "Pins IPFS data of a Kredits organization on an IPFS node",
   "main": "index.js",
   "scripts": {
@@ -20,7 +20,8 @@
   ],
   "license": "MIT",
   "dependencies": {
-    "@kredits/contracts": "git+https://gitea.kosmos.org/kredits/contracts#6e0ec87",
+    "@kredits/contracts": "^7.4.0",
+    "cli-progress": "^3.11.2",
     "debug": "^4.3.4",
     "yargs": "^17.6.0"
   },