Add error handling and retries for loading/pinning
This commit is contained in:
parent
4888d1be78
commit
c21512fa0b
|
@ -32,7 +32,7 @@ class IpfsPinner {
|
|||
for (const contract of contracts) {
|
||||
debug(`Pinning data from ${contract.constructor.name}...`);
|
||||
const itemCount = await contract.count;
|
||||
debug('Item count:', itemCount);
|
||||
debug(`${contract.constructor.name} item count:`, itemCount);
|
||||
let bar;
|
||||
|
||||
if (this.progressBars) {
|
||||
|
@ -68,21 +68,37 @@ class IpfsPinner {
|
|||
}
|
||||
|
||||
async _pinAllFromContract (contract, itemCount, progressBar) {
|
||||
const ipfsApi = this.ipfsApi;
|
||||
const progressBars = this.progressBars;
|
||||
const ids = [...Array(itemCount).keys()].map(i => i+1);
|
||||
const cids = [];
|
||||
const batchSize = 20;
|
||||
let position = 0;
|
||||
|
||||
while (position < itemCount) {
|
||||
const batchIds = ids.slice(position, position + batchSize);
|
||||
await Promise.all(batchIds.map(async id => {
|
||||
async function loadAndPin(id) {
|
||||
let cid;
|
||||
|
||||
try {
|
||||
const data = await contract.getData(id);
|
||||
debug(`Loaded ${contract.constructor.name} #${id}`);
|
||||
const cid = await this.ipfsApi.pin(data);
|
||||
cid = await ipfsApi.pin(data);
|
||||
debug(`Pinned ${contract.constructor.name} #${id} at ${cid}`);
|
||||
} catch(e) {
|
||||
debug(`Error while trying to load an pin ${contract.constructor.name} #${id}:`)
|
||||
debug(e);
|
||||
debug(`\nTrying again...`);
|
||||
loadAndPin(id);
|
||||
} finally {
|
||||
cids.push(cid);
|
||||
if (this.progressBars) { progressBar.increment(); }
|
||||
}));
|
||||
if (progressBars) { progressBar.increment(); }
|
||||
}
|
||||
}
|
||||
|
||||
while (position < itemCount) {
|
||||
const batchIds = ids.slice(position, position + batchSize);
|
||||
|
||||
await Promise.all(batchIds.map(async id => loadAndPin(id)));
|
||||
|
||||
position += batchSize;
|
||||
}
|
||||
|
||||
|
|
Loading…
Reference in New Issue