Merge pull request #190 from 67P/feature/improve_contribution_sync

Improve data sync

app/components/contribution-list/template.hbs

@@ -29,7 +29,7 @@
   </div>
 {{/if}}
 
-<ul class="contribution-list">
+<ul class="contribution-list {{if @loading 'loading'}}">
   {{#each this.contributionsFiltered as |contribution|}}
     <li role="button" data-contribution-id={{contribution.id}}
         {{action "openContributionDetails" contribution}}

app/components/contributor-list/component.js

@@ -2,12 +2,10 @@ import Component from '@ember/component';
 import { inject as service } from '@ember/service';
 
 export default Component.extend({
+  tagName: '',
 
   router: service(),
 
-  tagName: 'table',
-  classNames: 'contributor-list',
-
   selectedContributorId: null,
 
   actions: {
@@ -17,5 +15,4 @@ export default Component.extend({
     }
 
   }
-
 });

app/components/contributor-list/template.hbs

@@ -1,21 +1,25 @@
-<tbody>
-  {{#each @contributorList as |c|}}
-    <tr role="button"
-        onclick={{action "openContributorDetails" c.contributor}}
-        class="{{if (is-current-user c.contributor) "current-user"}} {{if (eq c.contributor.id @selectedContributorId) "selected"}}">
-      <td class="person">
-        <UserAvatar @contributor={{c.contributor}} /> {{c.contributor.name}}
-      </td>
-      <td class="kredits">
-        <span class="amount">
-          {{#if @showUnconfirmedKredits}}
-            {{c.amountTotal}}
-          {{else}}
-            {{c.amountConfirmed}}
-          {{/if}}
-        </span>
-        <span class="symbol">₭S</span>
-      </td>
-    </tr>
-  {{/each}}
-</tbody>
+<table class="contributor-list {{if @loading 'loading'}}">
+  <thead>
+  </thead>
+  <tbody>
+    {{#each @contributorList as |c|}}
+      <tr role="button"
+          onclick={{action "openContributorDetails" c.contributor}}
+          class="{{if (is-current-user c.contributor) "current-user"}} {{if (eq c.contributor.id @selectedContributorId) "selected"}}">
+        <td class="person">
+          <UserAvatar @contributor={{c.contributor}} /> {{c.contributor.name}}
+        </td>
+        <td class="kredits">
+          <span class="amount">
+            {{#if @showUnconfirmedKredits}}
+              {{c.amountTotal}}
+            {{else}}
+              {{c.amountConfirmed}}
+            {{/if}}
+          </span>
+          <span class="symbol">₭S</span>
+        </td>
+      </tr>
+    {{/each}}
+  </tbody>
+</table>

app/routes/application.js

@@ -24,10 +24,16 @@ export default Route.extend({
         this.kredits.addContractEventHandlers();
       })
       .then(() => {
-        if (this.kredits.contributorsNeedFetch) {
-          schedule('afterRender', this.kredits, this.kredits.fetchContributors);
+        if (this.kredits.contributorsNeedSync) {
+          schedule('afterRender', this.kredits.syncContributors,
+                   this.kredits.syncContributors.perform);
         }
-        schedule('afterRender', this.kredits, this.kredits.fetchContributions);
+        if (this.kredits.contributionsNeedSync) {
+          schedule('afterRender', this.kredits.syncContributions,
+                   this.kredits.syncContributions.perform);
+        }
+        schedule('afterRender', this.kredits.fetchMissingContributions,
+                 this.kredits.fetchMissingContributions.perform);
       });
   }
 });

app/services/kredits.js

@@ -1,6 +1,5 @@
 import ethers from 'ethers';
 import Kredits from 'kredits-contracts';
-import RSVP from 'rsvp';
 
 import Service from '@ember/service';
 import EmberObject from '@ember/object';
@@ -9,6 +8,8 @@ import { alias, notEmpty } from '@ember/object/computed';
 import { isEmpty, isPresent } from '@ember/utils';
 import { inject as service } from '@ember/service';
 
+import { task, taskGroup } from 'ember-concurrency';
+
 import groupBy from 'kredits-web/utils/group-by';
 import processContributorData from 'kredits-web/utils/process-contributor-data';
 import processContributionData from 'kredits-web/utils/process-contribution-data';
@@ -21,7 +22,6 @@ import Contribution from 'kredits-web/models/contribution'
 export default Service.extend({
 
   browserCache: service(),
-  contributorsNeedFetch: false,
 
   currentBlock: null,
   currentUserAccounts: null, // default to not having an account. this is the wen web3 is loaded.
@@ -34,6 +34,10 @@ export default Service.extend({
   currentUserIsCore: alias('currentUser.isCore'),
   hasAccounts: notEmpty('currentUserAccounts'),
 
+  // When data was loaded from cache, we need to fetch updates from the network
+  contributorsNeedSync: false,
+  contributionsNeedSync: false,
+
   contributionsUnconfirmed: computed('contributions.[]', 'currentBlock', function() {
     return this.contributions.filter(contribution => {
       return contribution.confirmedAt > this.currentBlock;
@@ -90,7 +94,7 @@ export default Service.extend({
   getEthProvider () {
     let ethProvider;
 
-    return new RSVP.Promise(async (resolve) => {
+    return new Promise(resolve => {
       function instantiateWithoutAccount () {
         console.debug('[kredits] Creating new instance from npm module class');
         console.debug(`[kredits] providerURL: ${config.web3ProviderUrl}`);
@@ -179,7 +183,7 @@ export default Service.extend({
     const numCachedContributors = await this.browserCache.contributors.length();
     if (numCachedContributors > 0) {
       await this.loadContributorsFromCache();
-      this.set('contributorsNeedFetch', true);
+      this.set('contributorsNeedSync', true);
     } else {
       await this.fetchContributors();
     }
@@ -187,6 +191,7 @@ export default Service.extend({
     const numCachedContributions = await this.browserCache.contributions.length();
     if (numCachedContributions > 0) {
       await this.loadContributionsFromCache();
+      this.set('contributionsNeedSync', true);
     } else {
       await this.fetchContributions({ page: { size: 30 } });
     }
@@ -226,12 +231,9 @@ export default Service.extend({
     console.debug(`[kredits] Fetching all contributors from the network`);
     return this.kredits.Contributor.all()
       .then(contributors => {
-        return contributors.map(data => {
-          const contributor = Contributor.create(processContributorData(data));
-          const loadedContributor = this.contributors.findBy('id', contributor.id);
-          if (loadedContributor) { this.contributors.removeObject(loadedContributor); }
-          this.contributors.pushObject(contributor);
-          return contributor;
+        return contributors.forEach(data => {
+          this.loadContributorFromData(data);
+          return;
         });
       })
       .then(() => {
@@ -239,6 +241,14 @@ export default Service.extend({
       });
   },
 
+  loadContributorFromData(data) {
+    const contributor = Contributor.create(processContributorData(data));
+    const loadedContributor = this.contributors.findBy('id', contributor.id);
+    if (loadedContributor) { this.contributors.removeObject(loadedContributor); }
+    this.contributors.pushObject(contributor);
+    return contributor;
+  },
+
   async cacheLoadedContributors () {
     for (const c of this.contributors) {
       await this.browserCache.contributors.setItem(c.id, c.serialize());
@@ -255,6 +265,10 @@ export default Service.extend({
     });
   },
 
+  syncContributors: task(function * () {
+    yield this.fetchContributors();
+  }),
+
   addContribution (attributes) {
     console.debug('[kredits] add contribution', attributes);
 
@@ -275,11 +289,7 @@ export default Service.extend({
     return this.kredits.Contribution.all(options)
       .then(contributions => {
         return contributions.map(data => {
-          const contribution = Contribution.create(processContributionData(data));
-          contribution.set('contributor', this.contributors.findBy('id', data.contributorId.toString()));
-          const loadedContribution = this.contributions.findBy('id', contribution.id);
-          if (loadedContribution) { this.contributions.removeObject(loadedContribution); }
-          this.contributions.pushObject(contribution);
+          const contribution = this.loadContributionFromData(data);
           return contribution;
         });
       })
@@ -293,6 +303,15 @@ export default Service.extend({
       });
   },
 
+  loadContributionFromData(data) {
+    const contribution = Contribution.create(processContributionData(data));
+    contribution.set('contributor', this.contributors.findBy('id', data.contributorId.toString()));
+    const loadedContribution = this.contributions.findBy('id', contribution.id);
+    if (loadedContribution) { this.contributions.removeObject(loadedContribution); }
+    this.contributions.pushObject(contribution);
+    return contribution;
+  },
+
   async cacheLoadedContributions () {
     for (const c of this.contributions) {
       await this.browserCache.contributions.setItem(c.id, c.serialize());
@@ -309,6 +328,72 @@ export default Service.extend({
     });
   },
 
+  contributionTasks: taskGroup().enqueue(),
+
+  syncContributions: task(function * () {
+    yield this.fetchNewContributions.perform();
+    yield this.syncUnconfirmedContributions.perform();
+  }).group('contributionTasks'),
+
+  fetchNewContributions: task(function * () {
+    const count = yield this.kredits.Contribution.functions.contributionsCount();
+    const lastKnownContributionId = Math.max.apply(null, this.contributions.mapBy('id'));
+    const toFetch = count - lastKnownContributionId;
+
+    if (toFetch > 0) {
+      console.debug(`[kredits] Fetching ${toFetch} new contributions`);
+      for (let id = lastKnownContributionId; id <= count; id++) {
+        const data = yield this.kredits.Contribution.getById(id);
+        const c = this.loadContributionFromData(data);
+        yield this.browserCache.contributions.setItem(c.id.toString(), c.serialize());
+      }
+    } else {
+      console.debug(`[kredits] No new contributions to fetch`);
+    }
+  }),
+
+  fetchMissingContributions: task(function * () {
+    const count = yield this.kredits.Contribution.functions.contributionsCount();
+    const allIds = [...Array(count+1).keys()];
+    allIds.shift(); // remove first item, which is 0
+    const loadedContributions = new Set(this.contributions.mapBy('id'));
+    const toFetch = allIds.filter(id => !loadedContributions.has(id));
+    if (toFetch.length === 0) {
+      console.debug(`[kredits] No contributions left to fetch`);
+      return;
+    }
+    console.debug(`[kredits] Fetching ${toFetch.length} past contributions`);
+    let countFetched = 0;
+
+    for (let id = count; id > 0; id--) {
+      if (loadedContributions.has(id)) {
+        continue;
+      } else {
+        const data = yield this.kredits.Contribution.getById(id);
+        const c = this.loadContributionFromData(data);
+        yield this.browserCache.contributions.setItem(c.id.toString(), c.serialize());
+        countFetched++;
+        if (countFetched % 20 === 0) {
+          console.debug(`[kredits] Fetched ${countFetched} more contributions`);
+        }
+      }
+    }
+    console.debug(`[kredits] Cached ${countFetched} past contributions`);
+  }).group('contributionTasks'),
+
+  syncUnconfirmedContributions: task(function * () {
+    if (this.contributionsUnconfirmed.length > 0) {
+      console.debug(`[kredits] Syncing unconfirmed contributions`);
+      for (const c of this.contributionsUnconfirmed) {
+        const data = yield this.kredits.Contribution.getById(c.id);
+        const contribution = this.loadContributionFromData(data);
+        yield this.browserCache.contributions.setItem(c.id.toString(), contribution.serialize());
+      }
+    } else {
+      console.debug(`[kredits] No unconfirmed contributions to sync`);
+    }
+  }),
+
   veto (contributionId) {
     console.debug('[kredits] veto against', contributionId);
     const contribution = this.contributions.findBy('id', contributionId);
@@ -323,14 +408,14 @@ export default Service.extend({
 
   getCurrentUser: computed('kredits.provider', 'currentUserAccounts.[]', function() {
     if (isEmpty(this.currentUserAccounts)) {
-      return RSVP.resolve();
+      return Promise.resolve();
     }
     return this.kredits.Contributor
       .functions.getContributorIdByAddress(this.currentUserAccounts.firstObject)
       .then((id) => {
         // check if the user is a contributor or not
         if (id === 0) {
-          return RSVP.resolve();
+          return Promise.resolve();
         } else {
           return this.kredits.Contributor.getById(id);
         }

app/styles/_sugar.scss

@@ -0,0 +1,17 @@
+@mixin loading-border-top {
+  &::before {
+    display: block;
+    width: 100%;
+    height: 1px;
+    content: '';
+    background-image: linear-gradient(45deg, rgba(255, 255, 255, 0.2) 40%, #68d7fb 60%, rgba(255, 255, 255, 0.2));
+    background-size: 200% 200%;
+    animation: kitt 2.5s linear infinite;
+  }
+}
+
+@keyframes kitt {
+  0%{ background-position: 0% 0%}
+  50%{ background-position: 100% 0%}
+  100%{ background-position: 0% 0%}
+}

app/styles/app.scss

@@ -88,6 +88,7 @@ section {
 
 @import "buttons";
 @import "forms";
+@import "sugar";
 
 @import "components/contribution-list";
 @import "components/contribution-details";

app/styles/components/_contribution-list.scss

@@ -114,4 +114,13 @@ ul.contribution-list {
     }
   }
 
+  &.loading {
+    @include loading-border-top;
+
+    li {
+      &:first-of-type {
+        border-top: none;
+      }
+    }
+  }
 }

app/styles/components/_contributor-list.scss

@@ -1,4 +1,5 @@
 table.contributor-list {
+  position: relative;
   width: 100%;
   border-collapse: collapse;
   margin-bottom: 1.5rem;
@@ -48,4 +49,11 @@ table.contributor-list {
       }
     }
   }
+
+  &.loading {
+    @include loading-border-top;
+    &::before {
+      position: absolute;
+    }
+  }
 }

app/templates/dashboard.hbs

@@ -29,7 +29,8 @@
       <div class="content">
         <ContributorList @contributorList={{this.kreditsToplist}}
                          @showUnconfirmedKredits={{this.showUnconfirmedKredits}}
-                         @selectedContributorId={{this.selectedContributorId}} />
+                         @selectedContributorId={{this.selectedContributorId}}
+                         @loading={{this.kredits.syncContributors.isRunning}} />
 
         <p class="stats">
           <span class="number">{{await this.kredits.totalKreditsEarned}}</span> kredits confirmed and issued to
@@ -58,7 +59,9 @@
     {{#if this.contributionsUnconfirmed}}
       <section id="contributions-unconfirmed">
         <header class="with-nav">
-          <h2>Latest Contributions</h2>
+          <h2>
+            Latest Contributions
+          </h2>
           <nav>
             <button type="button"
                     onclick={{action "toggleQuickFilterUnconfirmed"}}
@@ -78,7 +81,8 @@
                             @vetoContribution={{action "vetoContribution"}}
                             @contractInteractionEnabled={{this.kredits.hasAccounts}}
                             @selectedContributionId={{this.selectedContributionId}}
-                            @showQuickFilter={{this.showQuickFilterUnconfirmed}} />
+                            @showQuickFilter={{this.showQuickFilterUnconfirmed}}
+                            @loading={{this.kredits.syncContributions.isRunning}}/>
         </div>
       </section>
     {{/if}}

package-lock.json

@@ -10268,6 +10268,17 @@
         "semver": "^5.4.1"
       }
     },
+    "ember-concurrency": {
+      "version": "1.1.7",
+      "resolved": "https://registry.npmjs.org/ember-concurrency/-/ember-concurrency-1.1.7.tgz",
+      "integrity": "sha512-PtEJvB4wG8e5CEHRC9ILl2BxF6U/xlMOhfCji/x7FxNFB9M230Du86LAy+4/yOozZHyoARVuazABPUj02P+DmQ==",
+      "dev": true,
+      "requires": {
+        "ember-cli-babel": "^7.7.3",
+        "ember-compatibility-helpers": "^1.2.0",
+        "ember-maybe-import-regenerator": "^0.1.6"
+      }
+    },
     "ember-diff-attrs": {
       "version": "0.2.2",
       "resolved": "https://registry.npmjs.org/ember-diff-attrs/-/ember-diff-attrs-0.2.2.tgz",

package.json

@@ -48,6 +48,7 @@
     "ember-cli-sass": "^10.0.1",
     "ember-cli-sri": "^2.1.1",
     "ember-cli-uglify": "^3.0.0",
+    "ember-concurrency": "^1.1.7",
     "ember-export-application-global": "^2.0.1",
     "ember-fetch": "^8.0.1",
     "ember-flatpickr": "^2.16.2",