kredits/ipfs-pinner
9
1
Fork 0
Code Issues 2 Pull Requests 1 Actions Releases 9 Activity

Adapt code for new kredits release/contracts

Browse Source 
Also process loading and pinning functions in batches, to prevent stack
size errors on RSK nodes.
This commit is contained in:
Râu Cao
2022-11-01 15:02:18 +01:00
parent e12e798735
commit 4a3988137e
2 changed files with 42 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
lib/ipfs-pinner.js
View File

@@ -1,5 +1,16 @@
const debug = require('debug')('ipfs-pinner');
async function promiseAllInBatches(task, items, batchSize) {
let position = 0;
let results = [];
while (position < items.length) {
const itemsForBatch = items.slice(position, position + batchSize);
results = [...results, ...await Promise.allSettled(itemsForBatch.map(item => task(item)))];
position += batchSize;
}
return results;
}
class IpfsPinner {
constructor (kredits, ipfsApi) {
this.kredits = kredits;
@@ -9,10 +20,10 @@ class IpfsPinner {
async pinAll () {
const contributorHashes = await this._pinAllFromContract(this.kredits.Contributor);
const contributionHashes = await this._pinAllFromContract(this.kredits.Contribution);
const proposalHashes = await this._pinAllFromContract(this.kredits.Proposal);
// const proposalHashes = await this._pinAllFromContract(this.kredits.Proposal);
return contributorHashes.concat(contributionHashes)
.concat(proposalHashes);
return contributorHashes.concat(contributionHashes);
// .concat(proposalHashes);
}
monitor (callback) {
@@ -33,19 +44,25 @@ class IpfsPinner {
});
}
_pinAllFromContract (contract) {
async _pinAllFromContract (contract) {
debug(`Pinning data from ${contract.constructor.name}...`);
return contract.count.then(count => {
let promises = [...Array(count).keys()].map(i => {
let id = i + 1; // 0 => 1 - ids start with 1 and not with 0
debug(`Loading ${contract.constructor.name} #${id}`);
return contract.getData(id).then(data => {
debug(`Pinning ${contract.constructor.name} #${id}`);
return this.ipfsApi.pin(data);
});
const count = await contract.count;
debug('Item count:', count);
const ids = [...Array(count).keys()].map(i => i+1);
const cids = [];
async function loadAndPin (id) {
debug(`Loading ${contract.constructor.name} #${id}`);
return contract.getData(id).then(data => {
debug(`Pinning ${contract.constructor.name} #${id}`);
return this.ipfsApi.pin(data).then(cid => cids.push(cid));
});
return Promise.all(promises);
});
}
await promiseAllInBatches(loadAndPin.bind(this), ids, 100);
return cids;
}
}
module.exports = IpfsPinner;