693 lines
22 KiB
JavaScript
693 lines
22 KiB
JavaScript
import 'regenerator-runtime/runtime';
|
|
|
|
function pad (num) {
|
|
num = String(num);
|
|
if (num.length === 1) { num = "0" + num; }
|
|
return num;
|
|
};
|
|
|
|
function parseDate (date) {
|
|
return {
|
|
year: date.getUTCFullYear(),
|
|
month: pad( date.getUTCMonth() + 1 ),
|
|
day: pad( date.getUTCDate() )
|
|
};
|
|
};
|
|
|
|
function lowestNumberInListing(items) {
|
|
const sortedNumbers = Object.keys(items)
|
|
.map(i => parseInt(i))
|
|
.filter(i => !Number.isNaN(i))
|
|
.sort();
|
|
return sortedNumbers[0];
|
|
}
|
|
|
|
const ChatMessages = function (privateClient, publicClient) {
|
|
/**
|
|
* Schema: chat-messages/daily-archive
|
|
*
|
|
* Represents one calendar day of chat messages
|
|
*
|
|
* @example
|
|
* {
|
|
* "@context": "https://kosmos.org/ns/v2",
|
|
* "@id": "chat-messages/irc.libera.chat/channels/kosmos/",
|
|
* "@type": "ChatChannel",
|
|
* "service": {
|
|
* "domain": "irc.libera.chat",
|
|
* "protocol": "IRC",
|
|
* },
|
|
* "name": "#kosmos",
|
|
* "type": "room",
|
|
* "today": {
|
|
* "@id": "2015/01/01",
|
|
* "@type": "ChatLog",
|
|
* "messageType": "InstantMessage",
|
|
* "previous": "2014/12/31",
|
|
* "next": "2015/01/02",
|
|
* "messages": [
|
|
* { "date": "2015-06-05T17:35:28.454Z", "user": "hal8000", "text": "knock knock" },
|
|
* { "date": "2015-06-05T17:37:42.123Z", "user": "raucao", "text": "who's there?" },
|
|
* { "date": "2015-06-05T17:55:01.235Z", "user": "hal8000", "text": "HAL" }
|
|
* ]
|
|
* }
|
|
* }
|
|
*/
|
|
const archiveSchema = {
|
|
"type": "object",
|
|
"properties": {
|
|
"@context": {
|
|
"type": "string",
|
|
"default": "https://kosmos.org/ns/v2/chat-channel"
|
|
},
|
|
"@id": {
|
|
"type": "string",
|
|
"required": true
|
|
},
|
|
"@type": {
|
|
"type": "string",
|
|
"default": "ChatChannel",
|
|
"enum": ["ChatChannel"]
|
|
},
|
|
"service": {
|
|
"type": "object",
|
|
"properties": {
|
|
"domain": {
|
|
"type": "string",
|
|
"required": true
|
|
},
|
|
"protocol": {
|
|
"type": "string",
|
|
"required": true
|
|
}
|
|
}
|
|
},
|
|
"name": {
|
|
"type": "string",
|
|
"required": true
|
|
},
|
|
"type": {
|
|
"type": "string",
|
|
"required": true,
|
|
"enum": [ "room", "person" ]
|
|
},
|
|
"today": {
|
|
"type": "object",
|
|
"properties": {
|
|
"@id": {
|
|
"type": "string",
|
|
"pattern": "^[0-9]{4}\/[0-9]{2}\/[0-9]{2}$",
|
|
"required": true
|
|
},
|
|
"@type": {
|
|
"type": "string",
|
|
"default": "ChatLog",
|
|
"pattern": "^ChatLog$"
|
|
},
|
|
"messageType": {
|
|
"type": "string",
|
|
"default": "InstantMessage",
|
|
"pattern": "^InstantMessage$"
|
|
},
|
|
"previous": {
|
|
"type": "string",
|
|
"pattern": "^[0-9]{4}\/[0-9]{2}\/[0-9]{2}$"
|
|
},
|
|
"next": {
|
|
"type": "string",
|
|
"pattern": "^[0-9]{4}\/[0-9]{2}\/[0-9]{2}$"
|
|
},
|
|
"messages": {
|
|
"type": "array",
|
|
"required": true,
|
|
"items": {
|
|
"type": "object",
|
|
"properties": {
|
|
"date": {
|
|
"type": "string",
|
|
"format": "date-time"
|
|
},
|
|
"user": {
|
|
"type": "string"
|
|
},
|
|
"text": {
|
|
"type": "string"
|
|
},
|
|
"type": "string",
|
|
"default": "text",
|
|
"enum": [
|
|
"text",
|
|
"join",
|
|
"leave",
|
|
"action"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"required": []
|
|
};
|
|
|
|
privateClient.declareType("daily-archive", "https://kosmos.org/ns/v2/chat-channel", archiveSchema);
|
|
publicClient.declareType("daily-archive", "https://kosmos.org/ns/v2/chat-channel", archiveSchema);
|
|
|
|
/**
|
|
* Schema: chat-messages/daily-archive-meta
|
|
*
|
|
* Stores meta information about the daily archives
|
|
*
|
|
* @example
|
|
* {
|
|
* "@context": "https://kosmos.org/ns/v2",
|
|
* "@id": "chat-messages/irc.libera.chat/channels/kosmos/meta",
|
|
* "@type": "ChatChannelMeta",
|
|
* "first": "2009/01/03",
|
|
* "last": "2021/11/05"
|
|
* }
|
|
* }
|
|
*/
|
|
const archiveMetaSchema = {
|
|
"type": "object",
|
|
"properties": {
|
|
"@context": {
|
|
"type": "string",
|
|
"default": "https://kosmos.org/ns/v2/chat-channel-meta"
|
|
},
|
|
"@id": {
|
|
"type": "string",
|
|
},
|
|
"@type": {
|
|
"type": "string",
|
|
"default": "ChatChannelMeta",
|
|
"enum": ["ChatChannelMeta"]
|
|
},
|
|
"first": {
|
|
"type": "string",
|
|
"pattern": "^[0-9]{4}\/[0-9]{2}\/[0-9]{2}$"
|
|
},
|
|
"last": {
|
|
"type": "string",
|
|
"pattern": "^[0-9]{4}\/[0-9]{2}\/[0-9]{2}$"
|
|
}
|
|
},
|
|
"required": ["@id", "first", "last"]
|
|
};
|
|
|
|
privateClient.declareType("daily-archive-meta", "https://kosmos.org/ns/v2/chat-channel-meta", archiveMetaSchema);
|
|
publicClient.declareType("daily-archive-meta", "https://kosmos.org/ns/v2/chat-channel-meta", archiveMetaSchema);
|
|
|
|
/**
|
|
* A daily archive stores chat messages by calendar day.
|
|
*
|
|
* @class
|
|
*/
|
|
class DailyArchive {
|
|
/**
|
|
* @param {object} options
|
|
* @param {object} options.service
|
|
* @param {string} options.service.protocol - Type of chat service/protocol (e.g. "IRC", "XMPP", "Campfire", "Slack")
|
|
* @param {string} options.service.domain - Domain of the chat service (e.g. "irc.libera.chat", "kosmos.chat")
|
|
* @param {string} options.channelName - Name of room/channel (e.g. "#kosmos")
|
|
* @param {string} [options.channelType] - Type of channel ("room" or "person")
|
|
* @param {date} options.date - Date of archive day
|
|
* @param {boolean} [options.isPublic] - Store logs in public folder (defaults to false)
|
|
* @param {string} [options.previous] - Date of previous log file as `YYYY/MM/DD`. Looked up automatically when not given
|
|
* @param {string} [options.next] - Date of next log file as `YYYY/MM/DD`. looked up automatically when not given
|
|
*
|
|
* @example
|
|
* // IRC archive:
|
|
* const archive = new chatMessages.DailyArchive({
|
|
* service: {
|
|
* protocol: 'IRC',
|
|
* domain: 'irc.libera.chat',
|
|
* },
|
|
* channelName: '#kosmos-dev',
|
|
* channelType: 'room',
|
|
* date: new Date(),
|
|
* isPublic: true
|
|
* });
|
|
*
|
|
* // XMPP archive:
|
|
* const archive = new chatMessages.DailyArchive({
|
|
* service: {
|
|
* protocol: 'XMPP',
|
|
* domain: 'kosmos.chat',
|
|
* },
|
|
* channelName: 'kosmos-dev',
|
|
* channelType: 'room',
|
|
* date: new Date(),
|
|
* isPublic: false
|
|
* });
|
|
*
|
|
*/
|
|
constructor (options) {
|
|
//
|
|
// Defaults
|
|
//
|
|
options.isPublic = options.isPublic || false;
|
|
options.channelType = options.channelType || "room";
|
|
|
|
//
|
|
// Validate options
|
|
//
|
|
if (typeof options !== "object") {
|
|
throw "options must be an object";
|
|
}
|
|
if (typeof options.service !== "object" ||
|
|
typeof options.service.protocol !== "string" ||
|
|
typeof options.service.domain !== "string") {
|
|
throw "service must be an object containing at least service \"protocol\" and \"domain\"";
|
|
}
|
|
if (typeof options.channelName !== "string") {
|
|
throw "channelName must be a string";
|
|
}
|
|
if (!(options.date instanceof Date)) {
|
|
throw "date must be a date object";
|
|
}
|
|
if (typeof options.isPublic !== "boolean") {
|
|
throw "isPublic must be a boolean value";
|
|
}
|
|
|
|
/**
|
|
* @property {object} service
|
|
* @property {string} service.protocol - Type of chat service/protocol (e.g. "IRC", "XMPP", "campfire", "slack")
|
|
* @property {string} service.domain - Domain of the chat service (e.g. "irc.libera.chat", "kosmos.chat")
|
|
*/
|
|
this.service = options.service;
|
|
|
|
/**
|
|
* @property {string} channelName - Name of channel (e.g. "#kosmos")
|
|
*/
|
|
this.channelName = options.channelName;
|
|
|
|
/**
|
|
* @property {string} channelType - Type of channel ("room" or "person")
|
|
*/
|
|
this.channelType = options.channelType;
|
|
|
|
/**
|
|
* @property {string} date - Gregorian calendar date of the archive's content
|
|
*/
|
|
this.date = options.date;
|
|
|
|
/**
|
|
* @property {boolean} isPublic - `true` for public archives, `false` for private ones
|
|
*/
|
|
this.isPublic = options.isPublic || false;
|
|
|
|
/**
|
|
* @property {object} parsedDate - Contains padded year, month and day of date
|
|
* @property {string} year
|
|
* @property {string} month
|
|
* @property {string} day
|
|
*/
|
|
this.parsedDate = parseDate(this.date);
|
|
|
|
/**
|
|
* @property {string} dateId - Date string in the form of YYYY/MM/DD
|
|
*/
|
|
this.dateId = this.parsedDate.year+'/'+this.parsedDate.month+'/'+this.parsedDate.day;
|
|
|
|
/**
|
|
* @property {string} channelPath - Base directory path of the channel archives
|
|
*/
|
|
if (this.channelType === "room") {
|
|
// Normal chatroom
|
|
const channelName = this.channelName.replace(/#/,'');
|
|
this.channelPath = `${this.service.domain}/channels/${channelName}`;
|
|
} else {
|
|
// User direct messages
|
|
this.channelPath = `${this.service.domain}/users/${this.channelName}`;
|
|
}
|
|
|
|
/**
|
|
* @property {string} path - Path of the archive document
|
|
*/
|
|
this.path = `${this.channelPath}/${this.dateId}`;
|
|
|
|
/**
|
|
* @property {string} metaPath - Path of the channel's metadata document
|
|
*/
|
|
this.metaPath = `${this.channelPath}/meta`;
|
|
|
|
/**
|
|
* @property {object} client - Public or private remoteStorgage.js BaseClient
|
|
*/
|
|
this.client = this.isPublic ? publicClient : privateClient;
|
|
|
|
/**
|
|
* @property {string} previous - Date of previous log file as YYYY/MM/DD
|
|
*/
|
|
this.previous = options.previous;
|
|
|
|
/**
|
|
* @property {string} next - Date of next log file as YYYY/MM/DD
|
|
*/
|
|
this.next = options.next;
|
|
}
|
|
|
|
/*
|
|
* @param {object} message
|
|
* @param {string} message.timestamp - Timestamp of the message
|
|
* @param {string} message.from - The sender of the message
|
|
* @param {string} message.text - The message itself
|
|
* @param {string} message.type - Type of message (one of text, join, leave, action)
|
|
* @param {string} [message.id] - Unique ID of message. TODO implement
|
|
*
|
|
* @returns {Promise}
|
|
*/
|
|
addMessage (message) {
|
|
if (this.isPublic && !this.channelName.match(/^#/)) {
|
|
return Promise.resolve(false);
|
|
}
|
|
|
|
message.type = message.type || 'text';
|
|
|
|
return this.client.getObject(this.path).then((archive) => {
|
|
if (typeof archive === 'object') {
|
|
return this._updateDocument(archive, message);
|
|
} else {
|
|
return this._createDocument(message);
|
|
}
|
|
});
|
|
}
|
|
|
|
/*
|
|
* Like <addMessage>, but for multiple messages at once. Useful for bulk
|
|
* imports of messages.
|
|
*
|
|
* @param {Array} messages - Array of message objects (see params for addMessage)
|
|
* @param {boolean} overwrite - If true, creates a new archive file and overwrites the old one. Defaults to false.
|
|
*
|
|
* @returns {Promise}
|
|
*/
|
|
addMessages (messages, overwrite) {
|
|
if (this.isPublic && !this.channelName.match(/^#/)) {
|
|
return Promise.resolve(false);
|
|
}
|
|
|
|
overwrite = overwrite || false;
|
|
|
|
messages.forEach(function(message) {
|
|
message.type = message.type || 'text';
|
|
});
|
|
|
|
if (overwrite) {
|
|
return this._createDocument(messages);
|
|
} else {
|
|
return this.client.getObject(this.path).then((archive) => {
|
|
if (typeof archive === 'object') {
|
|
return this._updateDocument(archive, messages);
|
|
} else {
|
|
return this._createDocument(messages);
|
|
}
|
|
});
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Deletes the entire archive document from storage
|
|
*
|
|
* @returns {Promise}
|
|
*/
|
|
remove () {
|
|
// TODO when removing, if previous is set, but not next, it means the
|
|
// removed file is the last archive. Thus, set "last" to previous file.
|
|
return this.client.remove(this.path);
|
|
}
|
|
|
|
/*
|
|
* Updates and writes an existing archive document
|
|
*
|
|
* @returns {Promise}
|
|
*
|
|
* @private
|
|
*/
|
|
async _updateDocument (archive, messages) {
|
|
console.debug('[chat-messages] Updating archive document');
|
|
|
|
if (Array.isArray(messages)) {
|
|
messages.forEach(function(message) {
|
|
archive.today.messages.push(message);
|
|
});
|
|
} else {
|
|
archive.today.messages.push(messages);
|
|
}
|
|
|
|
return this._sync(archive);
|
|
}
|
|
|
|
/*
|
|
* Creates and writes a new archive document
|
|
*
|
|
* @returns {Promise}
|
|
*
|
|
* @private
|
|
*/
|
|
async _createDocument (messages) {
|
|
console.debug('[chat-messages] Creating new archive document');
|
|
const archive = this._buildArchiveObject();
|
|
|
|
if (Array.isArray(messages)) {
|
|
messages.forEach(message => {
|
|
archive.today.messages.push(message);
|
|
});
|
|
} else {
|
|
archive.today.messages.push(messages);
|
|
}
|
|
|
|
if (this.previous || this.next) {
|
|
// The app is handling previous/next keys itself
|
|
// That includes setting 'next' in the previous log file
|
|
if (this.previous) { archive.today.previous = this.previous; }
|
|
if (this.next) { archive.today.next = this.next; }
|
|
} else {
|
|
// Find and update previous archive, set 'previous' on this one
|
|
const previous = await this._updatePreviousArchive();
|
|
if (typeof previous === 'object') {
|
|
archive.today.previous = previous.today['@id'];
|
|
}
|
|
}
|
|
|
|
await this._sync(archive);
|
|
|
|
// TODO only write meta doc if argument is set on addMessages. This way
|
|
// we can avoid race conditions when syncing remote chat messages all at
|
|
// once for multiple days
|
|
await this._updateArchiveMetaDocument();
|
|
|
|
return;
|
|
}
|
|
|
|
/*
|
|
* Builds the object to be stored in remote storage
|
|
*
|
|
* @returns {object}
|
|
*
|
|
* @private
|
|
*/
|
|
_buildArchiveObject () {
|
|
const roomName = this.channelName.replace(/#/,'');
|
|
|
|
const archive = {
|
|
"@id": "chat-messages/"+this.service.domain+"/channels/"+roomName+"/",
|
|
"@type": "ChatChannel",
|
|
"service": this.service,
|
|
"name": this.channelName,
|
|
"type": this.channelType,
|
|
"today": {
|
|
"@id": this.dateId,
|
|
"@type": "ChatLog",
|
|
"messageType": "InstantMessage",
|
|
"messages": []
|
|
}
|
|
};
|
|
|
|
switch (this.service.protocol) {
|
|
case 'IRC':
|
|
if (!this.channelName.match(/^#/)) {
|
|
archive["@id"] = "chat-messages/"+this.service.domain+"/users/"+this.channelName+"/";
|
|
}
|
|
break;
|
|
case 'XMPP':
|
|
// XMPP-specific adjustments
|
|
break;
|
|
}
|
|
|
|
return archive;
|
|
}
|
|
|
|
/*
|
|
* Finds the previous archive document and updates its today.next value
|
|
*
|
|
* @returns {boolean|Promise}
|
|
*
|
|
* @private
|
|
*/
|
|
_updatePreviousArchive () {
|
|
return this._findPreviousArchive().then((archive) => {
|
|
if (typeof archive === 'object' && archive.today) {
|
|
archive.today.next = this.dateId;
|
|
const path = this.path.substring(0, this.path.length-this.dateId.length)+archive.today['@id'];
|
|
|
|
return this.client.storeObject('daily-archive', path, archive).then(() => {
|
|
console.debug('[chat-messages] Previous archive written to remote storage at', path);
|
|
return archive;
|
|
});
|
|
} else {
|
|
console.debug('[chat-messages] Previous archive not found');
|
|
return false;
|
|
}
|
|
});
|
|
}
|
|
|
|
/*
|
|
* Returns the previous archive document
|
|
*
|
|
* @returns {Promise}
|
|
*
|
|
* @private
|
|
*/
|
|
_findPreviousArchive () {
|
|
const monthPath = this.path.substring(0, this.path.length-2);
|
|
const yearPath = this.path.substring(0, this.path.length-5);
|
|
const basePath = this.path.substring(0, this.path.length-10);
|
|
|
|
return this.client.getListing(monthPath).then((listing) => {
|
|
const days = Object.keys(listing).map((i) => parseInt(i)).map((i) => {
|
|
return (i < parseInt(this.parsedDate.day)) ? i : null;
|
|
}).filter(function(i){ return i != null; });
|
|
|
|
if (days.length > 0) {
|
|
const day = pad(Math.max(...days).toString());
|
|
return this.client.getObject(monthPath+day);
|
|
}
|
|
|
|
// Find last day in previous month
|
|
return this.client.getListing(yearPath).then((listing) => {
|
|
const months = Object.keys(listing).map((i) => parseInt(i.substr(0,2))).map((i) => {
|
|
return (i < parseInt(this.parsedDate.month)) ? i : null;
|
|
}).filter(function(i){ return i != null; });
|
|
|
|
if (months.length > 0) {
|
|
const month = pad(Math.max(...months).toString());
|
|
|
|
return this.client.getListing(yearPath+month+'/').then((listing) => {
|
|
const days = Object.keys(listing).map((i) => parseInt(i));
|
|
const day = pad(Math.max(...days).toString());
|
|
return this.client.getObject(yearPath+month+'/'+day);
|
|
});
|
|
} else {
|
|
// Find last month and day in previous year
|
|
return this.client.getListing(basePath).then((listing) => {
|
|
|
|
const years = Object.keys(listing).map((i) => parseInt(i.substr(0,4))).map((i) => {
|
|
return (i < parseInt(this.parsedDate.year)) ? i : null;
|
|
}).filter(function(i){ return i != null; });
|
|
|
|
if (years.length > 0) {
|
|
const year = Math.max(...years).toString();
|
|
|
|
return this.client.getListing(basePath+year+'/').then((listing) => {
|
|
const months = Object.keys(listing).map((i) => parseInt(i.substr(0,2)));
|
|
const month = pad(Math.max(...months).toString());
|
|
|
|
return this.client.getListing(basePath+year+'/'+month+'/').then((listing) => {
|
|
const days = Object.keys(listing).map((i) => parseInt(i));
|
|
const day = pad(Math.max(...days).toString());
|
|
return this.client.getObject(basePath+year+'/'+month+'/'+day);
|
|
});
|
|
});
|
|
} else {
|
|
return false;
|
|
}
|
|
});
|
|
}
|
|
});
|
|
});
|
|
}
|
|
|
|
async _updateArchiveMetaDocument () {
|
|
const meta = await this.client.getObject(this.metaPath);
|
|
if (typeof meta !== 'object') {
|
|
return this._createArchiveMetaDocument();
|
|
}
|
|
|
|
// Only update document if current date is newer than known "last"
|
|
if (Date.parse(meta.last.replace('/','-')) > Date.parse(this.date)) {
|
|
console.debug('[chat-messages]', 'Updating meta document for channel');
|
|
meta.last = this.dateId;
|
|
await this.client.storeObject('daily-archive-meta', this.metaPath, meta);
|
|
}
|
|
|
|
return;
|
|
}
|
|
|
|
async _createArchiveMetaDocument () {
|
|
console.debug('[chat-messages]', 'Creating new meta document for channel');
|
|
// When creating a new meta doc, we need to find the oldest archive,
|
|
// because older versions of the module did not write a meta doc.
|
|
const first = await this._findFirstArchive();
|
|
const roomName = this.channelName.replace(/#/,'');
|
|
|
|
const meta = {
|
|
'@id': `chat-messages/${this.service.domain}/channels/${roomName}/meta`,
|
|
'@type': 'ChatChannelMeta',
|
|
first: first,
|
|
last: this.dateId // TODO might have to search for last?
|
|
};
|
|
|
|
return this.client.storeObject('daily-archive-meta', this.metaPath, meta)
|
|
.then(() => console.debug('[chat-messages]', 'Meta document written to remote storage'))
|
|
.catch(e => {
|
|
console.log('[chat-messages]', `Failed to store ${this.metaPath}`);
|
|
console.error(e);
|
|
});
|
|
}
|
|
|
|
async _findFirstArchive () {
|
|
console.debug('[chat-messages]', 'Finding first archive for channel');
|
|
const years = await this.client.getListing(`${this.channelPath}/`);
|
|
const year = lowestNumberInListing(years);
|
|
const months = await this.client.getListing(`${this.channelPath}/${year}/`);
|
|
const month = lowestNumberInListing(months);
|
|
const days = await this.client.getListing(`${this.channelPath}/${year}/${pad(month)}/`);
|
|
const day = lowestNumberInListing(days);
|
|
const firstId = `${year}/${pad(month)}/${pad(day)}`;
|
|
console.debug('[chat-messages]', 'First is', firstId);
|
|
return firstId;
|
|
}
|
|
|
|
/*
|
|
* Write archive document
|
|
*
|
|
* @returns {Promise}
|
|
*
|
|
* @private
|
|
*/
|
|
async _sync (obj) {
|
|
console.debug(`[chat-messages] Writing archive object with ${obj.today.messages.length} messages`);
|
|
|
|
return this.client.storeObject('daily-archive', this.path, obj).then(function(){
|
|
console.debug('[chat-messages] Archive written to remote storage');
|
|
return true;
|
|
},function(error){
|
|
console.warn('[chat-messages] Error trying to store object', error);
|
|
return error;
|
|
});
|
|
}
|
|
};
|
|
|
|
return {
|
|
exports: {
|
|
DailyArchive,
|
|
privateClient,
|
|
publicClient
|
|
}
|
|
};
|
|
};
|
|
|
|
export default { name: 'chat-messages', builder: ChatMessages };
|