import 'regenerator-runtime/runtime'; function pad (num) { num = String(num); if (num.length === 1) { num = "0" + num; } return num; }; function parseDate (date) { return { year: date.getUTCFullYear(), month: pad( date.getUTCMonth() + 1 ), day: pad( date.getUTCDate() ) }; }; function lowestNumberInListing(items) { const sortedNumbers = Object.keys(items) .map(i => parseInt(i)) .filter(i => !Number.isNaN(i)) .sort(); return sortedNumbers[0]; } const ChatMessages = function (privateClient, publicClient) { /** * Schema: chat-messages/daily-archive * * Represents one calendar day of chat messages * * @example * { * "@context": "https://kosmos.org/ns/v2", * "@id": "chat-messages/irc.libera.chat/channels/kosmos/", * "@type": "ChatChannel", * "service": { * "domain": "irc.libera.chat", * "protocol": "IRC", * }, * "name": "#kosmos", * "type": "room", * "today": { * "@id": "2015/01/01", * "@type": "ChatLog", * "messageType": "InstantMessage", * "previous": "2014/12/31", * "next": "2015/01/02", * "messages": [ * { "date": "2015-06-05T17:35:28.454Z", "user": "hal8000", "text": "knock knock" }, * { "date": "2015-06-05T17:37:42.123Z", "user": "raucao", "text": "who's there?" }, * { "date": "2015-06-05T17:55:01.235Z", "user": "hal8000", "text": "HAL" } * ] * } * } */ const archiveSchema = { "type": "object", "properties": { "@context": { "type": "string", "default": "https://kosmos.org/ns/v2/chat-channel" }, "@id": { "type": "string", "required": true }, "@type": { "type": "string", "default": "ChatChannel", "enum": ["ChatChannel"] }, "service": { "type": "object", "properties": { "domain": { "type": "string", "required": true }, "protocol": { "type": "string", "required": true } } }, "name": { "type": "string", "required": true }, "type": { "type": "string", "required": true, "enum": [ "room", "person" ] }, "today": { "type": "object", "properties": { "@id": { "type": "string", "pattern": "^[0-9]{4}\/[0-9]{2}\/[0-9]{2}$", "required": true }, "@type": { "type": "string", "default": "ChatLog", "pattern": "^ChatLog$" }, "messageType": { "type": "string", "default": "InstantMessage", "pattern": "^InstantMessage$" }, "previous": { "type": "string", "pattern": "^[0-9]{4}\/[0-9]{2}\/[0-9]{2}$" }, "next": { "type": "string", "pattern": "^[0-9]{4}\/[0-9]{2}\/[0-9]{2}$" }, "messages": { "type": "array", "required": true, "items": { "type": "object", "properties": { "date": { "type": "string", "format": "date-time" }, "user": { "type": "string" }, "text": { "type": "string" }, "type": "string", "default": "text", "enum": [ "text", "join", "leave", "action" ] } } } } } }, "required": [] }; privateClient.declareType("daily-archive", "https://kosmos.org/ns/v2/chat-channel", archiveSchema); publicClient.declareType("daily-archive", "https://kosmos.org/ns/v2/chat-channel", archiveSchema); /** * Schema: chat-messages/daily-archive-meta * * Stores meta information about the daily archives * * @example * { * "@context": "https://kosmos.org/ns/v2", * "@id": "chat-messages/irc.libera.chat/channels/kosmos/meta", * "@type": "ChatChannelMeta", * "first": "2009/01/03", * "last": "2021/11/05" * } * } */ const archiveMetaSchema = { "type": "object", "properties": { "@context": { "type": "string", "default": "https://kosmos.org/ns/v2/chat-channel-meta" }, "@id": { "type": "string", }, "@type": { "type": "string", "default": "ChatChannelMeta", "enum": ["ChatChannelMeta"] }, "first": { "type": "string", "pattern": "^[0-9]{4}\/[0-9]{2}\/[0-9]{2}$" }, "last": { "type": "string", "pattern": "^[0-9]{4}\/[0-9]{2}\/[0-9]{2}$" } }, "required": ["@id", "first", "last"] }; privateClient.declareType("daily-archive-meta", "https://kosmos.org/ns/v2/chat-channel-meta", archiveMetaSchema); publicClient.declareType("daily-archive-meta", "https://kosmos.org/ns/v2/chat-channel-meta", archiveMetaSchema); /** * A daily archive stores chat messages by calendar day. * * @class */ class DailyArchive { /** * @param {object} options * @param {object} options.service * @param {string} options.service.protocol - Type of chat service/protocol (e.g. "IRC", "XMPP", "Campfire", "Slack") * @param {string} options.service.domain - Domain of the chat service (e.g. "irc.libera.chat", "kosmos.chat") * @param {string} options.channelName - Name of room/channel (e.g. "#kosmos") * @param {string} [options.channelType] - Type of channel ("room" or "person") * @param {date} options.date - Date of archive day * @param {boolean} [options.isPublic] - Store logs in public folder (defaults to false) * @param {string} [options.previous] - Date of previous log file as `YYYY/MM/DD`. Looked up automatically when not given * @param {string} [options.next] - Date of next log file as `YYYY/MM/DD`. looked up automatically when not given * * @example * // IRC archive: * const archive = new chatMessages.DailyArchive({ * service: { * protocol: 'IRC', * domain: 'irc.libera.chat', * }, * channelName: '#kosmos-dev', * channelType: 'room', * date: new Date(), * isPublic: true * }); * * // XMPP archive: * const archive = new chatMessages.DailyArchive({ * service: { * protocol: 'XMPP', * domain: 'kosmos.chat', * }, * channelName: 'kosmos-dev', * channelType: 'room', * date: new Date(), * isPublic: false * }); * */ constructor (options) { // // Defaults // options.isPublic = options.isPublic || false; options.channelType = options.channelType || "room"; // // Validate options // if (typeof options !== "object") { throw "options must be an object"; } if (typeof options.service !== "object" || typeof options.service.protocol !== "string" || typeof options.service.domain !== "string") { throw "service must be an object containing at least service \"protocol\" and \"domain\""; } if (typeof options.channelName !== "string") { throw "channelName must be a string"; } if (!(options.date instanceof Date)) { throw "date must be a date object"; } if (typeof options.isPublic !== "boolean") { throw "isPublic must be a boolean value"; } /** * @property {object} service * @property {string} service.protocol - Type of chat service/protocol (e.g. "IRC", "XMPP", "campfire", "slack") * @property {string} service.domain - Domain of the chat service (e.g. "irc.libera.chat", "kosmos.chat") */ this.service = options.service; /** * @property {string} channelName - Name of channel (e.g. "#kosmos") */ this.channelName = options.channelName; /** * @property {string} channelType - Type of channel ("room" or "person") */ this.channelType = options.channelType; /** * @property {string} date - Gregorian calendar date of the archive's content */ this.date = options.date; /** * @property {boolean} isPublic - `true` for public archives, `false` for private ones */ this.isPublic = options.isPublic || false; /** * @property {object} parsedDate - Contains padded year, month and day of date * @property {string} year * @property {string} month * @property {string} day */ this.parsedDate = parseDate(this.date); /** * @property {string} dateId - Date string in the form of YYYY/MM/DD */ this.dateId = this.parsedDate.year+'/'+this.parsedDate.month+'/'+this.parsedDate.day; /** * @property {string} channelPath - Base directory path of the channel archives */ if (this.channelType === "room") { // Normal chatroom const channelName = this.channelName.replace(/#/,''); this.channelPath = `${this.service.domain}/channels/${channelName}`; } else { // User direct messages this.channelPath = `${this.service.domain}/users/${this.channelName}`; } /** * @property {string} path - Path of the archive document */ this.path = `${this.channelPath}/${this.dateId}`; /** * @property {string} metaPath - Path of the channel's metadata document */ this.metaPath = `${this.channelPath}/meta`; /** * @property {object} client - Public or private remoteStorgage.js BaseClient */ this.client = this.isPublic ? publicClient : privateClient; /** * @property {string} previous - Date of previous log file as YYYY/MM/DD */ this.previous = options.previous; /** * @property {string} next - Date of next log file as YYYY/MM/DD */ this.next = options.next; } /* * @param {object} message * @param {string} message.timestamp - Timestamp of the message * @param {string} message.from - The sender of the message * @param {string} message.text - The message itself * @param {string} message.type - Type of message (one of text, join, leave, action) * @param {string} [message.id] - Unique ID of message. TODO implement * * @returns {Promise} */ addMessage (message) { if (this.isPublic && !this.channelName.match(/^#/)) { return Promise.resolve(false); } message.type = message.type || 'text'; return this.client.getObject(this.path).then((archive) => { if (typeof archive === 'object') { return this._updateDocument(archive, message); } else { return this._createDocument(message); } }); } /* * Like , but for multiple messages at once. Useful for bulk * imports of messages. * * @param {Array} messages - Array of message objects (see params for addMessage) * @param {boolean} overwrite - If true, creates a new archive file and overwrites the old one. Defaults to false. * * @returns {Promise} */ addMessages (messages, overwrite) { if (this.isPublic && !this.channelName.match(/^#/)) { return Promise.resolve(false); } overwrite = overwrite || false; messages.forEach(function(message) { message.type = message.type || 'text'; }); if (overwrite) { return this._createDocument(messages); } else { return this.client.getObject(this.path).then((archive) => { if (typeof archive === 'object') { return this._updateDocument(archive, messages); } else { return this._createDocument(messages); } }); } } /* * Deletes the entire archive document from storage * * @returns {Promise} */ remove () { // TODO when removing, if previous is set, but not next, it means the // removed file is the last archive. Thus, set "last" to previous file. return this.client.remove(this.path); } /* * Updates and writes an existing archive document * * @returns {Promise} * * @private */ async _updateDocument (archive, messages) { console.debug('[chat-messages] Updating archive document'); if (Array.isArray(messages)) { messages.forEach(function(message) { archive.today.messages.push(message); }); } else { archive.today.messages.push(messages); } return this._sync(archive); } /* * Creates and writes a new archive document * * @returns {Promise} * * @private */ async _createDocument (messages) { console.debug('[chat-messages] Creating new archive document'); const archive = this._buildArchiveObject(); if (Array.isArray(messages)) { messages.forEach(message => { archive.today.messages.push(message); }); } else { archive.today.messages.push(messages); } if (this.previous || this.next) { // The app is handling previous/next keys itself // That includes setting 'next' in the previous log file if (this.previous) { archive.today.previous = this.previous; } if (this.next) { archive.today.next = this.next; } } else { // Find and update previous archive, set 'previous' on this one const previous = await this._updatePreviousArchive(); if (typeof previous === 'object') { archive.today.previous = previous.today['@id']; } } await this._sync(archive); // TODO only write meta doc if argument is set on addMessages. This way // we can avoid race conditions when syncing remote chat messages all at // once for multiple days await this._updateArchiveMetaDocument(); return; } /* * Builds the object to be stored in remote storage * * @returns {object} * * @private */ _buildArchiveObject () { const roomName = this.channelName.replace(/#/,''); const archive = { "@id": "chat-messages/"+this.service.domain+"/channels/"+roomName+"/", "@type": "ChatChannel", "service": this.service, "name": this.channelName, "type": this.channelType, "today": { "@id": this.dateId, "@type": "ChatLog", "messageType": "InstantMessage", "messages": [] } }; switch (this.service.protocol) { case 'IRC': if (!this.channelName.match(/^#/)) { archive["@id"] = "chat-messages/"+this.service.domain+"/users/"+this.channelName+"/"; } break; case 'XMPP': // XMPP-specific adjustments break; } return archive; } /* * Finds the previous archive document and updates its today.next value * * @returns {boolean|Promise} * * @private */ _updatePreviousArchive () { return this._findPreviousArchive().then((archive) => { if (typeof archive === 'object' && archive.today) { archive.today.next = this.dateId; const path = this.path.substring(0, this.path.length-this.dateId.length)+archive.today['@id']; return this.client.storeObject('daily-archive', path, archive).then(() => { console.debug('[chat-messages] Previous archive written to remote storage at', path); return archive; }); } else { console.debug('[chat-messages] Previous archive not found'); return false; } }); } /* * Returns the previous archive document * * @returns {Promise} * * @private */ _findPreviousArchive () { const monthPath = this.path.substring(0, this.path.length-2); const yearPath = this.path.substring(0, this.path.length-5); const basePath = this.path.substring(0, this.path.length-10); return this.client.getListing(monthPath).then((listing) => { const days = Object.keys(listing).map((i) => parseInt(i)).map((i) => { return (i < parseInt(this.parsedDate.day)) ? i : null; }).filter(function(i){ return i != null; }); if (days.length > 0) { const day = pad(Math.max(...days).toString()); return this.client.getObject(monthPath+day); } // Find last day in previous month return this.client.getListing(yearPath).then((listing) => { const months = Object.keys(listing).map((i) => parseInt(i.substr(0,2))).map((i) => { return (i < parseInt(this.parsedDate.month)) ? i : null; }).filter(function(i){ return i != null; }); if (months.length > 0) { const month = pad(Math.max(...months).toString()); return this.client.getListing(yearPath+month+'/').then((listing) => { const days = Object.keys(listing).map((i) => parseInt(i)); const day = pad(Math.max(...days).toString()); return this.client.getObject(yearPath+month+'/'+day); }); } else { // Find last month and day in previous year return this.client.getListing(basePath).then((listing) => { const years = Object.keys(listing).map((i) => parseInt(i.substr(0,4))).map((i) => { return (i < parseInt(this.parsedDate.year)) ? i : null; }).filter(function(i){ return i != null; }); if (years.length > 0) { const year = Math.max(...years).toString(); return this.client.getListing(basePath+year+'/').then((listing) => { const months = Object.keys(listing).map((i) => parseInt(i.substr(0,2))); const month = pad(Math.max(...months).toString()); return this.client.getListing(basePath+year+'/'+month+'/').then((listing) => { const days = Object.keys(listing).map((i) => parseInt(i)); const day = pad(Math.max(...days).toString()); return this.client.getObject(basePath+year+'/'+month+'/'+day); }); }); } else { return false; } }); } }); }); } async _updateArchiveMetaDocument () { const meta = await this.client.getObject(this.metaPath); if (typeof meta !== 'object') { return this._createArchiveMetaDocument(); } // Only update document if current date is newer than known "last" if (Date.parse(meta.last.replace('/','-')) > Date.parse(this.date)) { console.debug('[chat-messages]', 'Updating meta document for channel'); meta.last = this.dateId; await this.client.storeObject('daily-archive-meta', this.metaPath, meta); } return; } async _createArchiveMetaDocument () { console.debug('[chat-messages]', 'Creating new meta document for channel'); // When creating a new meta doc, we need to find the oldest archive, // because older versions of the module did not write a meta doc. const first = await this._findFirstArchive(); const roomName = this.channelName.replace(/#/,''); const meta = { '@id': `chat-messages/${this.service.domain}/channels/${roomName}/meta`, '@type': 'ChatChannelMeta', first: first, last: this.dateId // TODO might have to search for last? }; return this.client.storeObject('daily-archive-meta', this.metaPath, meta) .then(() => console.debug('[chat-messages]', 'Meta document written to remote storage')) .catch(e => { console.log('[chat-messages]', `Failed to store ${this.metaPath}`); console.error(e); }); } async _findFirstArchive () { console.debug('[chat-messages]', 'Finding first archive for channel'); const years = await this.client.getListing(`${this.channelPath}/`); const year = lowestNumberInListing(years); const months = await this.client.getListing(`${this.channelPath}/${year}/`); const month = lowestNumberInListing(months); const days = await this.client.getListing(`${this.channelPath}/${year}/${pad(month)}/`); const day = lowestNumberInListing(days); const firstId = `${year}/${pad(month)}/${pad(day)}`; console.debug('[chat-messages]', 'First is', firstId); return firstId; } /* * Write archive document * * @returns {Promise} * * @private */ async _sync (obj) { console.debug(`[chat-messages] Writing archive object with ${obj.today.messages.length} messages`); return this.client.storeObject('daily-archive', this.path, obj).then(function(){ console.debug('[chat-messages] Archive written to remote storage'); return true; },function(error){ console.warn('[chat-messages] Error trying to store object', error); return error; }); } }; return { exports: { DailyArchive, privateClient, publicClient } }; }; export default { name: 'chat-messages', builder: ChatMessages };