raucao/marco
1
0
Fork 0
Code Issues 18 Pull Requests Actions Releases 20 Activity

Migrate OSM cache to localForage/IndexedDB

Browse Source 
We don't want to hit limits with localStorage, and it's non-blocking
This commit is contained in:
raucao
2026-08-19 10:39:58 -06:00
parent 45f6f898fa
commit 61242420d1
10 changed files with 1421 additions and 1231 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/services/contributions.js
+85 -56
View File
@@ -2,7 +2,7 @@ import Service, { service } from '@ember/service';
import { tracked } from '@glimmer/tracking';
import { groupPhotoContributions } from '../utils/contributions';
const NAME_CACHE_KEY = 'marco:contributions:name_cache';
const NAME_CACHE_STORE = 'contributions-name-cache';
/**
* Orchestrates loading the user's own Nostr contributions, grouping them into
@@ -11,54 +11,90 @@ const NAME_CACHE_KEY = 'marco:contributions:name_cache';
* The flow is:
* 1. Subscribe to `nostrData.store.timeline(...)` for the user's kind 360 events.
* 2. Group events into contribution entries via `groupPhotoContributions`.
* 3. Resolve place names immediately from bookmarks / name cache / OSM cache.
* 3. Resolve place names from bookmarks (sync, instant) on the first render,
* then asynchronously from the IndexedDB name cache and OSM cache.
* 4. For unresolved names, batch-fetch from the OSM API in the background.
* 5. Any items that can't be resolved get a fallback name so they don't stay
* stuck in a "Loading…" state forever.
* 6. Update `@tracked items` so the UI renders progressively.
*
* The persistent name cache is stored in IndexedDB (via the `localForage`
* service) with one key per `placeIdentifier`, so partial updates don't
* require re-serializing the whole map.
*/
export default class ContributionsService extends Service {
@service nostrData;
@service nostrAuth;
@service storage;
@service osm;
@service localForage;
@tracked items = [];
_sub = null;
_pendingBatchPromise = null;
_lastBatchSignature = '';
_nameCache = new Map();
_unresolvable = new Set();
constructor() {
super(...arguments);
this._loadNameCache();
/**
* Async name resolution. Checks, in order:
* 1. Bookmarks (sync, instant).
* 2. The persistent IndexedDB name cache (per-entry key).
* 3. The OSM service's IndexedDB cache (from place detail visits).
*
* @param {object} entry A contribution entry with `osmType`, `osmId`, and `placeIdentifier`.
* @returns {Promise<string|null>}
*/
async _resolveCachedName(entry) {
// 1. Try bookmarks (instant)
const bookmark = this.storage.findPlaceById(entry.osmId);
if (bookmark?.title) return bookmark.title;
// 2. Try the persistent name cache (IndexedDB, per-entry, survives sessions)
const cachedName = await this.localForage.get(
NAME_CACHE_STORE,
entry.placeIdentifier
);
if (cachedName) return cachedName;
// 3. Try OSM IndexedDB cache (from place detail visits)
const cached = await this.osm.getCachedOsmObject(
entry.osmType,
entry.osmId
);
return cached?.title || null;
}
_loadNameCache() {
if (typeof localStorage === 'undefined') return;
try {
const raw = localStorage.getItem(NAME_CACHE_KEY);
if (raw) {
const obj = JSON.parse(raw);
if (obj && typeof obj === 'object') {
this._nameCache = new Map(Object.entries(obj));
/**
* Asynchronously resolves names for still-loading entries from the caches.
* Falls through to `_maybeBatchFetchNames` for anything that remains
* unresolved. Fire-and-forget from `_updateItems` so the list renders
* immediately with bookmark-resolved names.
*
* @param {Array} entries
* @returns {Promise<void>}
*/
async _resolveFromCache(entries) {
const pending = entries.filter((e) => e.placeNameLoading);
if (pending.length === 0) {
this._maybeBatchFetchNames(this.items);
return;
}
await Promise.all(
pending.map(async (entry) => {
const name = await this._resolveCachedName(entry);
if (name) {
entry.placeName = name;
entry.placeNameLoading = false;
}
}
} catch {
// ignore malformed cache
}
}
})
);
_saveNameCache() {
if (typeof localStorage === 'undefined') return;
try {
const obj = Object.fromEntries(this._nameCache);
localStorage.setItem(NAME_CACHE_KEY, JSON.stringify(obj));
} catch (e) {
console.debug('[contributions] Failed to persist name cache', e);
}
// Re-render with whatever resolved, then kick off the network batch for
// entries that are still loading.
this.items = [...this.items];
this._maybeBatchFetchNames(this.items);
}
/**
@@ -109,36 +145,21 @@ export default class ContributionsService extends Service {
// 1. Group events into contribution entries (newest-first)
const entries = groupPhotoContributions(events);
// 2. Resolve place names: preserve previously-resolved names, then check
// bookmarks, the name cache, and the OSM service cache.
// 2. Bookmark lookup is synchronous — resolve those immediately so the
// first render shows bookmarked place names without a "Loading…" flicker.
for (const entry of entries) {
const cached = this._resolveCachedName(entry);
if (cached) {
entry.placeName = cached;
const bookmark = this.storage.findPlaceById(entry.osmId);
if (bookmark?.title) {
entry.placeName = bookmark.title;
entry.placeNameLoading = false;
}
}
this.items = entries;
// 3. Trigger a background batch fetch for any still-unresolved names.
// De-duplicate so we don't re-fetch the same set while a fetch is in-flight.
this._maybeBatchFetchNames(entries);
}
_resolveCachedName(entry) {
// 1. Try bookmarks (instant)
const bookmark = this.storage.findPlaceById(entry.osmId);
if (bookmark?.title) return bookmark.title;
// 2. Try the persistent name cache (instant, survives across sessions)
if (this._nameCache.has(entry.placeIdentifier)) {
return this._nameCache.get(entry.placeIdentifier);
}
// 3. Try OSM localStorage cache (instant, from place detail visits)
const cached = this.osm.getCachedOsmObject(entry.osmType, entry.osmId);
return cached?.title || null;
// 3. Async-resolve remaining entries from the IndexedDB name cache and OSM
// cache, then fall through to the network batch for the rest.
void this._resolveFromCache(entries);
}
_isUnresolvable(entry) {
@@ -177,7 +198,7 @@ export default class ContributionsService extends Service {
this._pendingBatchPromise = this._batchResolveNames(unresolved)
.then((nameMap) => {
// Merge resolved names back into the current `items` and the name cache.
// Merge resolved names back into the current `items`.
for (const item of this.items) {
if (!item.placeNameLoading) continue;
if (nameMap.has(item.placeIdentifier)) {
@@ -192,7 +213,6 @@ export default class ContributionsService extends Service {
item.placeNameLoading = false;
}
}
this._saveNameCache();
// Trigger a re-render
this.items = [...this.items];
})
@@ -206,7 +226,6 @@ export default class ContributionsService extends Service {
item.placeNameLoading = false;
}
}
this._saveNameCache();
this.items = [...this.items];
})
.finally(() => {
@@ -224,7 +243,7 @@ export default class ContributionsService extends Service {
// Re-check cache in case it was populated between the trigger and now
for (const entry of entries) {
const cached = this._resolveCachedName(entry);
const cached = await this._resolveCachedName(entry);
if (cached) {
nameMap.set(entry.placeIdentifier, cached);
} else {
@@ -235,16 +254,26 @@ export default class ContributionsService extends Service {
if (toFetch.length === 0) return nameMap;
const places = await this.osm.fetchOsmObjectsBatch(toFetch);
// Persist each newly-resolved name to the IndexedDB name cache as its own
// entry (per-placeIdentifier key) so we don't re-fetch next session.
const writePromises = [];
for (const entry of entries) {
if (nameMap.has(entry.placeIdentifier)) continue;
const cacheKey = `${entry.osmType}:${entry.osmId}`;
const place = places.get(cacheKey);
if (place?.title) {
nameMap.set(entry.placeIdentifier, place.title);
this._nameCache.set(entry.placeIdentifier, place.title);
writePromises.push(
this.localForage.set(
NAME_CACHE_STORE,
entry.placeIdentifier,
place.title
)
);
}
}
this._saveNameCache();
await Promise.all(writePromises);
return nameMap;
}
}
app/services/local-forage.js
+55
View File
@@ -0,0 +1,55 @@
import localforage from 'localforage';
import Service from '@ember/service';
const DB_NAME = 'marco';
/**
* Thin async-only wrapper around `localforage` that exposes namespaced
* object stores under a single IndexedDB database (`marco`).
*
* Each "store" is a `localforage` instance created with a unique
* `storeName`, so callers can keep data isolated (e.g. OSM cache vs.
* contributions name cache) without managing their own connections.
*
* All methods return Promises — IndexedDB is inherently async, so callers
* must `await` every read/write.
*/
export default class LocalForageService extends Service {
_instances = new Map();
_instance(storeName) {
let instance = this._instances.get(storeName);
if (!instance) {
instance = localforage.createInstance({
name: DB_NAME,
storeName,
});
this._instances.set(storeName, instance);
}
return instance;
}
async get(storeName, key) {
return this._instance(storeName).getItem(key);
}
async set(storeName, key, value) {
return this._instance(storeName).setItem(key, value);
}
async remove(storeName, key) {
return this._instance(storeName).removeItem(key);
}
async keys(storeName) {
return this._instance(storeName).keys();
}
async clear(storeName) {
return this._instance(storeName).clear();
}
async iterate(storeName, fn) {
return this._instance(storeName).iterate(fn);
}
}
app/services/osm.js
+34 -32
View File
@@ -4,16 +4,18 @@ import { getCategoryById } from '../utils/poi-categories';
export default class OsmService extends Service {
@service settings;
@service localForage;
controller = null;
cachedResults = null;
lastQueryKey = null;
cachedPlaces = new Map();
// Long-term cache for OSM place metadata, persisted to localStorage so that
// names and basic info survive across sessions and can be rendered instantly
// without waiting on the OSM API. Entries are refreshed in the background.
static CACHE_KEY_PREFIX = 'marco:osm_cache:';
// Long-term cache for OSM place metadata, persisted to IndexedDB (via the
// `localForage` service) so that names and basic info survive across
// sessions and can be rendered instantly without waiting on the OSM API.
// Entries are refreshed in the background.
static STORE_NAME = 'osm-cache';
static CACHE_TTL_MS = 7 * 24 * 60 * 60 * 1000; // 7 days
static IN_MEMORY_TTL_MS = 10000; // 10 seconds
@@ -21,27 +23,24 @@ export default class OsmService extends Service {
return `${osmType}:${osmId}`;
}
_readLocalCache(osmType, osmId) {
if (typeof localStorage === 'undefined') return null;
const key = `${OsmService.CACHE_KEY_PREFIX}${osmType}:${osmId}`;
async _readLocalCache(osmType, osmId) {
const key = this._buildCacheKey(osmType, osmId);
try {
const raw = localStorage.getItem(key);
const raw = await this.localForage.get(OsmService.STORE_NAME, key);
if (!raw) return null;
const parsed = JSON.parse(raw);
const parsed = typeof raw === 'string' ? JSON.parse(raw) : raw;
if (
!parsed ||
typeof parsed.timestamp !== 'number' ||
Date.now() - parsed.timestamp > OsmService.CACHE_TTL_MS
) {
localStorage.removeItem(key);
await this.localForage.remove(OsmService.STORE_NAME, key);
return null;
}
return parsed.data;
} catch {
try {
localStorage.removeItem(
`${OsmService.CACHE_KEY_PREFIX}${osmType}:${osmId}`
);
await this.localForage.remove(OsmService.STORE_NAME, key);
} catch {
// ignore
}
@@ -49,31 +48,32 @@ export default class OsmService extends Service {
}
}
_writeLocalCache(osmType, osmId, data) {
if (typeof localStorage === 'undefined' || !data) return;
const key = `${OsmService.CACHE_KEY_PREFIX}${osmType}:${osmId}`;
async _writeLocalCache(osmType, osmId, data) {
if (!data) return;
const key = this._buildCacheKey(osmType, osmId);
try {
localStorage.setItem(
key,
JSON.stringify({ data, timestamp: Date.now() })
);
await this.localForage.set(OsmService.STORE_NAME, key, {
data,
timestamp: Date.now(),
});
} catch (e) {
console.debug('[osm] Failed to write localStorage cache entry', e);
console.debug('[osm] Failed to write IndexedDB cache entry', e);
}
}
/**
* Synchronous lookup for an OSM place. Checks the short-lived in-memory cache
* first, then the persistent localStorage cache. Returns `null` if not cached.
* Async lookup for an OSM place. Checks the short-lived in-memory cache
* first, then the persistent IndexedDB cache. Returns `null` if not
* cached.
*
* Use this for instant rendering (e.g. place names in a list) and fall back to
* `fetchOsmObject` for a fresh fetch + background refresh.
* Use this for instant rendering (e.g. place names in a list) and fall
* back to `fetchOsmObject` for a fresh fetch + background refresh.
*
* @param {string} osmType 'node' | 'way' | 'relation'
* @param {string} osmId
* @returns {object|null} Normalized OSM place data
* @returns {Promise<object|null>} Normalized OSM place data
*/
getCachedOsmObject(osmType, osmId) {
async getCachedOsmObject(osmType, osmId) {
if (!osmType || !osmId) return null;
const cacheKey = this._buildCacheKey(osmType, osmId);
@@ -103,7 +103,9 @@ export default class OsmService extends Service {
this.cachedPlaces.delete(cacheKey);
}, OsmService.IN_MEMORY_TTL_MS);
this._writeLocalCache(osmType, osmId, data);
// Fire-and-forget the persistent write — the in-memory cache covers the
// next immediate read, and the IndexedDB write happens in the background.
void this._writeLocalCache(osmType, osmId, data);
}
cancelAll() {
@@ -336,12 +338,12 @@ out center;
return cached.data;
}
// If we have a persistent (localStorage) cache entry, return it immediately and
// If we have a persistent (IndexedDB) cache entry, return it immediately and
// kick off a background refresh. This keeps the UI snappy while still ensuring
// the cache is updated with the latest OSM data.
const localCached = this._readLocalCache(osmType, osmId);
const localCached = await this._readLocalCache(osmType, osmId);
if (localCached) {
console.debug(`Using localStorage cached OSM object for ${cacheKey}`);
console.debug(`Using IndexedDB cached OSM object for ${cacheKey}`);
// Refresh in the background, but don't block the caller.
this._refreshOsmObject(osmId, osmType, cacheKey).catch((e) => {
console.debug('[osm] Background refresh failed for', cacheKey, e);
@@ -415,7 +417,7 @@ out center;
for (const { osmType, osmId } of items) {
if (!osmType || !osmId) continue;
const cacheKey = this._buildCacheKey(osmType, osmId);
const cached = this.getCachedOsmObject(osmType, osmId);
const cached = await this.getCachedOsmObject(osmType, osmId);
if (cached) {
result.set(cacheKey, cached);
} else {