2.0.1

Reviewed-on: #10

README.md

@@ -1,3 +1,5 @@
+[![npm](https://img.shields.io/npm/v/@kredits/ipfs-pinner.svg)](https://www.npmjs.com/package/@kredits/ipfs-pinner)
+
 # Kredits IPFS Pinner
 
 This tool pins the IPFS documents of a Kredits organization on any IPFS node.

index.js

@@ -12,7 +12,7 @@ const argv = require('yargs')
     host: 'localhost',
     port: '5001',
     protocol: 'http',
-    watch: true,
+    watch: false,
     progress: false,
     bootstrapNode: `${defaultPeers[0].Addrs[0]}/ipfs/${defaultPeers[0].ID}`
   })
@@ -39,44 +39,45 @@ const ipfsConfig = {
 debug(`IPFS node:`, ipfsConfig);
 
 (async () => {
-  try {
+  const kredits = await Kredits.for(
-    const kredits = await Kredits.for(
+    { rpcUrl: argv.rpcUrl },
-      { rpcUrl: argv.rpcUrl },
+    { ipfsConfig: ipfsConfig }
-      { ipfsConfig: ipfsConfig }
+  ).init().catch(e => {
-    ).init();
+    console.log('Failed to initialize Kredits:');
-
+    console.log(e.message);
-    // Check the connection to the IPFS client
-    // TODO redesign IPFS wrapper API and do not use an internal attribute
-    const ipfsApi = kredits.ipfs._ipfsAPI;
-
-    await ipfsApi.id();
-
-    debug(`Connecting to known IPFS node ${argv.bootstrapNode}`);
-    await ipfsApi.swarm.connect(argv.bootstrapNode);
-
-    const ipfsPinner = new IpfsPinner(kredits, {
-      progress: argv.progress
-    });
-
-    await ipfsPinner.pinAll().then(cids => {
-      console.log(`\nSuccessfully pinned ${cids.length} documents`)
-    });
-
-    if (argv.watch) {
-      console.log('\nWatching contract events for new documents...');
-
-      ipfsPinner.watch(pin => {
-        console.log('Pinned a new document:', pin[0]["hash"]);
-      });
-    } else {
-      process.exit(0);
-    }
-
-    // TODO Add new deployment/DAO/org ID or all contract proxy addresses
-    // console.log(`Subscribed to DAO: ${kredits.Kernel.contract.address}`);
-  } catch(e) {
-    console.log('Failed to start');
-    console.log(e);
     process.exit(1);
+  });
+
+  // TODO redesign IPFS wrapper API and do not use an internal attribute
+  const ipfsApi = kredits.ipfs._ipfsAPI;
+
+  await ipfsApi.id().catch(e => {
+    console.log('Failed to initialize IPFS:');
+    console.log(e.message);
+    process.exit(1);
+  });
+
+  debug(`Connecting to known IPFS node ${argv.bootstrapNode}`);
+  await ipfsApi.swarm.connect(argv.bootstrapNode);
+
+  const ipfsPinner = new IpfsPinner(kredits, {
+    progress: argv.progress
+  });
+
+  await ipfsPinner.pinAll().then(cids => {
+    console.log(`\nSuccessfully pinned ${cids.length} documents`)
+  });
+
+  if (argv.watch) {
+    console.log('\nWatching contract events for new documents...');
+
+    ipfsPinner.watch(pin => {
+      console.log('Pinned a new document:', pin[0]["hash"]);
+    });
+  } else {
+    process.exit(0);
   }
+
+  // TODO Add new deployment/DAO/org ID or all contract proxy addresses
+  // console.log(`Subscribed to DAO: ${kredits.Kernel.contract.address}`);
 })();

lib/ipfs-pinner.js

@@ -1,6 +1,10 @@
 const debug = require('debug')('ipfs-pinner');
 const cliProgress = require('cli-progress');
 
+function sleep(ms) {
+  return new Promise(resolve => setTimeout(resolve, ms));
+}
+
 class IpfsPinner {
   constructor (kredits, options={}) {
     this.kredits = kredits;
@@ -32,7 +36,7 @@ class IpfsPinner {
     for (const contract of contracts) {
       debug(`Pinning data from ${contract.constructor.name}...`);
       const itemCount = await contract.count;
-      debug('Item count:', itemCount);
+      debug(`${contract.constructor.name} item count:`, itemCount);
       let bar;
 
       if (this.progressBars) {
@@ -46,6 +50,9 @@ class IpfsPinner {
 
     await Promise.all(promises);
 
+    // Avoid console output race condition with progress bars finishing update
+    if (this.progressBars) await sleep(1000);
+
     return cids;
   }
 
@@ -68,21 +75,37 @@ class IpfsPinner {
   }
 
   async _pinAllFromContract (contract, itemCount, progressBar) {
+    const ipfsApi = this.ipfsApi;
+    const progressBars = this.progressBars;
     const ids = [...Array(itemCount).keys()].map(i => i+1);
     const cids = [];
     const batchSize = 20;
     let position = 0;
 
-    while (position < itemCount) {
+    async function loadAndPin(id) {
-      const batchIds = ids.slice(position, position + batchSize);
+      let cid;
-      await Promise.all(batchIds.map(async id => {
+
+      try {
         const data = await contract.getData(id);
         debug(`Loaded ${contract.constructor.name} #${id}`);
-        const cid = await this.ipfsApi.pin(data);
+        cid = await ipfsApi.pin(data);
         debug(`Pinned ${contract.constructor.name} #${id} at ${cid}`);
+      } catch(e) {
+        debug(`Error while trying to load an pin ${contract.constructor.name} #${id}:`)
+        debug(e);
+        debug(`\nTrying again...`);
+        loadAndPin(id);
+      } finally {
         cids.push(cid);
-        if (this.progressBars) { progressBar.increment(); }
+        if (progressBars) { progressBar.increment(); }
-      }));
+      }
+    }
+
+    while (position < itemCount) {
+      const batchIds = ids.slice(position, position + batchSize);
+
+      await Promise.all(batchIds.map(async id => loadAndPin(id)));
+
       position += batchSize;
     }
 

package-lock.json

@@ -1,15 +1,15 @@
 {
-  "name": "@kosmos/kredits-ipfs-pinner",
+  "name": "@kredits/ipfs-pinner",
-  "version": "1.2.0",
+  "version": "2.0.1",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
-      "name": "@kosmos/kredits-ipfs-pinner",
+      "name": "@kredits/ipfs-pinner",
-      "version": "1.2.0",
+      "version": "2.0.1",
       "license": "MIT",
       "dependencies": {
-        "@kredits/contracts": "git+https://gitea.kosmos.org/kredits/contracts#6e0ec87",
+        "@kredits/contracts": "^7.0.0",
         "cli-progress": "^3.11.2",
         "debug": "^4.3.4",
         "yargs": "^17.6.0"
@@ -721,9 +721,9 @@
       "integrity": "sha512-yOTK5WiXFDNAitPByMabE365aEEzFHgSUSgAssbJWt7BZ80HQSVu8XWrQiTbFbCkoIBmXwPP/RoxgXJQVgZTFQ=="
     },
     "node_modules/@kredits/contracts": {
-      "version": "7.0.0-beta.0",
+      "version": "7.0.0",
-      "resolved": "git+https://gitea.kosmos.org/kredits/contracts#6e0ec8741e61b51fb5c9c636da4e8d3610d090ac",
+      "resolved": "https://registry.npmjs.org/@kredits/contracts/-/contracts-7.0.0.tgz",
-      "license": "MIT",
+      "integrity": "sha512-UITEkP3njFNI2WS7v5ivGE3ruFwdWPWuJZrhBXBEAZbtmr1t/p1K7jkmmjyLDUeKXJ/udMlH6oQMCgh7P/aHNg==",
       "dependencies": {
         "@kosmos/schemas": "^3.1.0",
         "ethers": "^5.4.7",
@@ -2245,8 +2245,9 @@
       "integrity": "sha512-yOTK5WiXFDNAitPByMabE365aEEzFHgSUSgAssbJWt7BZ80HQSVu8XWrQiTbFbCkoIBmXwPP/RoxgXJQVgZTFQ=="
     },
     "@kredits/contracts": {
-      "version": "git+https://gitea.kosmos.org/kredits/contracts#6e0ec8741e61b51fb5c9c636da4e8d3610d090ac",
+      "version": "7.0.0",
-      "from": "@kredits/contracts@git+https://gitea.kosmos.org/kredits/contracts#6e0ec87",
+      "resolved": "https://registry.npmjs.org/@kredits/contracts/-/contracts-7.0.0.tgz",
+      "integrity": "sha512-UITEkP3njFNI2WS7v5ivGE3ruFwdWPWuJZrhBXBEAZbtmr1t/p1K7jkmmjyLDUeKXJ/udMlH6oQMCgh7P/aHNg==",
       "requires": {
         "@kosmos/schemas": "^3.1.0",
         "ethers": "^5.4.7",

package.json

@@ -1,6 +1,6 @@
 {
-  "name": "@kosmos/kredits-ipfs-pinner",
+  "name": "@kredits/ipfs-pinner",
-  "version": "1.2.0",
+  "version": "2.0.1",
   "description": "Pins IPFS data of a Kredits organization on an IPFS node",
   "main": "index.js",
   "scripts": {
@@ -20,7 +20,7 @@
   ],
   "license": "MIT",
   "dependencies": {
-    "@kredits/contracts": "git+https://gitea.kosmos.org/kredits/contracts#6e0ec87",
+    "@kredits/contracts": "^7.0.0",
     "cli-progress": "^3.11.2",
     "debug": "^4.3.4",
     "yargs": "^17.6.0"