Add error handling and retries for loading/pinning

This commit is contained in:
Râu Cao 2022-11-02 12:58:07 +01:00
parent 4888d1be78
commit c21512fa0b
Signed by: raucao
GPG Key ID: 15E65F399D084BA9
1 changed files with 23 additions and 7 deletions

View File

@ -32,7 +32,7 @@ class IpfsPinner {
for (const contract of contracts) {
debug(`Pinning data from ${contract.constructor.name}...`);
const itemCount = await contract.count;
debug('Item count:', itemCount);
debug(`${contract.constructor.name} item count:`, itemCount);
let bar;
if (this.progressBars) {
@ -68,21 +68,37 @@ class IpfsPinner {
}
async _pinAllFromContract (contract, itemCount, progressBar) {
const ipfsApi = this.ipfsApi;
const progressBars = this.progressBars;
const ids = [...Array(itemCount).keys()].map(i => i+1);
const cids = [];
const batchSize = 20;
let position = 0;
while (position < itemCount) {
const batchIds = ids.slice(position, position + batchSize);
await Promise.all(batchIds.map(async id => {
async function loadAndPin(id) {
let cid;
try {
const data = await contract.getData(id);
debug(`Loaded ${contract.constructor.name} #${id}`);
const cid = await this.ipfsApi.pin(data);
cid = await ipfsApi.pin(data);
debug(`Pinned ${contract.constructor.name} #${id} at ${cid}`);
} catch(e) {
debug(`Error while trying to load an pin ${contract.constructor.name} #${id}:`)
debug(e);
debug(`\nTrying again...`);
loadAndPin(id);
} finally {
cids.push(cid);
if (this.progressBars) { progressBar.increment(); }
}));
if (progressBars) { progressBar.increment(); }
}
}
while (position < itemCount) {
const batchIds = ids.slice(position, position + batchSize);
await Promise.all(batchIds.map(async id => loadAndPin(id)));
position += batchSize;
}