diff --git a/src/lib/workers/export.js b/src/lib/workers/export.js index d78abfc..025f4b4 100644 --- a/src/lib/workers/export.js +++ b/src/lib/workers/export.js @@ -2,34 +2,35 @@ const { expose } = require('threads/worker'); const Cryptr = require('cryptr'); const { decrypt } = new Cryptr(process.env.ENCRYPTION_KEY); -function decryptIfExists(encrypted) { - if (encrypted) return decrypt(encrypted); - return null; -} - expose({ exportTicket(ticket) { - if (ticket.closedReason) ticket.closedReason = decrypt(ticket.closedReason); - if (ticket.feedback?.comment) ticket.feedback.comment = decrypt(ticket.feedback.comment); - if (ticket.topic) ticket.topic = decrypt(ticket.topic); - ticket.archivedMessages = ticket.archivedMessages.map(message => { - message.content = decryptIfExists(message.content); + message.content &&= decrypt(message.content); return message; }); ticket.archivedUsers = ticket.archivedUsers.map(user => { - user.displayName = decryptIfExists(user.displayName); - user.username = decryptIfExists(user.username); + user.displayName &&= decrypt(user.displayName); + user.username &&= decrypt(user.username); return user; }); + if (ticket.feedback) { + // why is feedback the only one with a guild relation? 😕 + delete ticket.feedback.guildId; + ticket.feedback.comment &&= decrypt(ticket.feedback.comment); + } + + ticket.closedReason &&= decrypt(ticket.closedReason); + + delete ticket.guildId; + ticket.questionAnswers = ticket.questionAnswers.map(answer => { - if (answer.value) answer.value = decryptIfExists(answer.value); + answer.value &&= decrypt(answer.value); return answer; }); - delete ticket.guildId; + ticket.topic &&= decrypt(ticket.topic); return JSON.stringify(ticket); }, diff --git a/src/lib/workers/import.js b/src/lib/workers/import.js new file mode 100644 index 0000000..425760c --- /dev/null +++ b/src/lib/workers/import.js @@ -0,0 +1,115 @@ +const { expose } = require('threads/worker'); +const Cryptr = require('cryptr'); +const { encrypt } = new Cryptr(process.env.ENCRYPTION_KEY); + +expose({ + importTicket(stringified, guildId, categoryMap) { + const ticket = JSON.parse(stringified); + + ticket.archivedChannels = { + create: ticket.archivedChannels.map(user => { + delete user.ticketId; + return user; + }), + }; + + ticket.archivedUsers = { + create: ticket.archivedUsers.map(user => { + delete user.ticketId; + user.displayName &&= encrypt(user.displayName); + user.username &&= encrypt(user.username); + return user; + }), + }; + + ticket.archivedRoles = { + create: ticket.archivedRoles.map(user => { + delete user.ticketId; + return user; + }), + }; + + const messages = ticket.archivedMessages.map(message => { + // messages don't need to be wrapped in {create} + message.content &&= encrypt(message.content); + return message; + }); + delete ticket.archivedMessages; + + ticket.category = { connect: { id: categoryMap.get(ticket.categoryId) } }; + delete ticket.categoryId; + + if (ticket.claimedById) { + ticket.claimedBy = { + connectOrCreate: { + create: { id: ticket.claimedById }, + where: { id: ticket.claimedById }, + }, + }; + } + delete ticket.claimedById; + + if (ticket.closedById) { + ticket.closedBy = { + connectOrCreate: { + create: { id: ticket.closedById }, + where: { id: ticket.closedById }, + }, + }; + } + delete ticket.closedById; + + if (ticket.createdById) { + ticket.createdBy = { + connectOrCreate: { + create: { id: ticket.createdById }, + where: { id: ticket.createdById }, + }, + }; + } + delete ticket.createdById; + + ticket.closedReason &&= encrypt(ticket.closedReason); + + if (ticket.feedback) { + ticket.feedback.guild = { connect: { id: guildId } }; + ticket.feedback.comment &&= encrypt(ticket.feedback.comment); + if (ticket.feedback.userId) { + ticket.feedback.user = { + connectOrCreate: { + create: { id: ticket.feedback.userId }, + where: { id: ticket.feedback.userId }, + }, + }; + delete ticket.feedback.userId; + } + ticket.feedback = { create: ticket.feedback }; + } else { + ticket.feedback = undefined; + } + + ticket.guild = { connect: { id: guildId } }; + delete ticket.guildId; // shouldn't exist but make sure + + if (ticket.questionAnswers.length) { + ticket.questionAnswers = { + create: ticket.questionAnswers.map(async answer => { + answer.value &&= encrypt(answer.value); + return answer; + }), + }; + } else { + ticket.questionAnswers = undefined; + } + + if (ticket.referencesTicketId) { + ticket.referencesTicket = { connect: { id: ticket.referencesTicketId } }; + } + delete ticket.referencesTicketId; + + ticket.topic &&= encrypt(ticket.topic); + + return [ticket, messages]; + + }, +}); diff --git a/src/routes/api/admin/guilds/[guild]/export.js b/src/routes/api/admin/guilds/[guild]/export.js index 4c4bdf2..89658f1 100644 --- a/src/routes/api/admin/guilds/[guild]/export.js +++ b/src/routes/api/admin/guilds/[guild]/export.js @@ -6,6 +6,8 @@ const { const { Readable } = require('node:stream'); const { cpus } = require('node:os'); const archiver = require('archiver'); +const { iconURL } = require('../../../../../lib/misc'); +const pkg = require('../../../../../../package.json'); // ! ceiL: at least 1 const poolSize = Math.ceil(cpus().length / 4); @@ -26,6 +28,26 @@ module.exports.get = fastify => ({ client.log.info(`${member.user.username} requested an export of "${guild.name}"`); + // TODO: sign so the importer can ensure files haven't been added (important for attachments) + const archive = archiver('zip', { + comment: JSON.stringify({ + exportedAt: new Date().toISOString(), + exportedFromClientId: client.user.id, + originalGuildId: id, + originalGuildName: guild.name, + version: pkg.version, + }), + }); + + archive.on('warning', err => { + if (err.code === 'ENOENT') client.log.warn(err); + else throw err; + }); + + archive.on('error', err => { + throw err; + }); + const settings = await client.prisma.guild.findUnique({ include: { categories: { include: { questions: true } }, @@ -34,6 +56,8 @@ module.exports.get = fastify => ({ where: { id }, }); + delete settings.id; + settings.categories = settings.categories.map(c => { delete c.guildId; return c; @@ -45,7 +69,6 @@ module.exports.get = fastify => ({ }); const ticketsStream = Readable.from(ticketsGenerator()); - async function* ticketsGenerator() { try { let done = false; @@ -72,21 +95,23 @@ module.exports.get = fastify => ({ } // ! map (parallel) not for...of (serial) yield* batch.map(async ticket => (await pool.queue(worker => worker.exportTicket(ticket)) + '\n')); + // Readable.from(AsyncGenerator) seems to be faster than pushing to a Readable with an empty `read()` function + // for (const ticket of batch) { + // pool + // .queue(worker => worker.exportTicket(ticket)) + // .then(string => ticketsStream.push(string + '\n')); + // } } while (!done); } finally { ticketsStream.push(null); // ! extremely important } } - const archive = archiver('zip', { - comment: JSON.stringify({ - exportedAt: new Date().toISOString(), - originalGuildId: id, - }), - }) - .append(JSON.stringify(settings), { name: 'settings.json' }) - .append(ticketsStream, { name: 'tickets.jsonl' }); - archive.finalize(); + const icon = await fetch(iconURL(guild)); + archive.append(Readable.from(icon.body), { name: 'icon.png' }); + archive.append(JSON.stringify(settings), { name: 'settings.json' }); + archive.append(ticketsStream, { name: 'tickets.jsonl' }); + archive.finalize(); // ! do not await const cleanGuildName = guild.name.replace(/\W/g, '_').replace(/_+/g, '_'); const fileName = `tickets-${cleanGuildName}-${new Date().toISOString().slice(0, 10)}`; diff --git a/src/routes/api/admin/guilds/[guild]/import.js b/src/routes/api/admin/guilds/[guild]/import.js new file mode 100644 index 0000000..a85766f --- /dev/null +++ b/src/routes/api/admin/guilds/[guild]/import.js @@ -0,0 +1,186 @@ + +const { + spawn, + Pool, + Worker, +} = require('threads'); +const { cpus } = require('node:os'); +const unzipper = require('unzipper'); +const { createInterface } = require('node:readline'); +const pkg = require('../../../../../../package.json'); + +// ! ceiL: at least 1 +const poolSize = Math.ceil(cpus().length / 4); +const pool = Pool(() => spawn(new Worker('../../../../../lib/workers/import.js')), { size: poolSize }); + +function parseJSON(string) { + try { + return JSON.parse(string); + } catch { + return null; + } +} + +// put would be better but forms can only get or post +module.exports.post = fastify => ({ + /** + * + * @param {import('fastify').FastifyRequest} req + * @param {import('fastify').FastifyReply} res + */ + handler: async (req, res) => { + /** @type {import('client')} */ + const client = req.routeOptions.config.client; + const id = req.params.guild; + const guild = client.guilds.cache.get(id); + const member = await guild.members.fetch(req.user.id); + + client.log.info(`${member.user.username} is importing data to "${guild.name}"`); + + client.keyv.delete(`cache/stats/guild:${id}`); + + const [zFile] = await req.saveRequestFiles({ + limits: { + fields: 1, + files: 1, + }, + }); + + res.raw.writeHead(200, { 'Content-Type': 'text/html; charset=utf-8' }); + + const userLog = { + _write(style1, style2, prefix, string) { + res.raw.write(`

[${prefix}] ${string}

`); + }, + error(string) { + this._write('text-red-500 font-bold', 'text-red-700 dark:text-red-200', 'ERROR', string); + }, + info(string) { + this._write('text-cyan-500', 'text-cyan-700 dark:text-cyan-200', 'INFO', string); + }, + success(string) { + this._write('text-green-500', 'text-green-700 dark:text-green-200', 'SUCCESS', string); + }, + warn(string) { + this._write('text-orange-500', 'text-orange-700 dark:text-orange-200', 'WARN', string); + }, + }; + + try { + // comment needs to be less than 512B + const zip = await unzipper.Open.file(zFile.filepath, { tailSize: 512 }); + const { files } = zip; + const comment = parseJSON(zip.comment); + client.log.info('Import comment', comment); + if (comment) { + userLog.info(`v${comment.version} -> v${pkg.version}`); + } else { + userLog.warn('Comment is not parsable'); + } + + userLog.info('Reading settings.json'); + // `settingsJSON` is frozen, `settings` can be mutated + const settingsJSON = JSON.parse(await files.find(f => f.path === 'settings.json').buffer()); + Object.freeze(settingsJSON); + const settings = structuredClone(settingsJSON); + const { categories } = settings; + delete settings.categories; // this also mutates `settings + + userLog.info('Importing general settings and tags'); + await client.prisma.$transaction([ + client.prisma.guild.delete({ + select: { id: true }, + where: { id }, + }), + client.prisma.guild.create({ + data: { + ...settings, + id, + tags: { + createMany: { + data: settings.tags.map(tag => { + delete tag.id; + return tag; + }), + }, + }, + }, + // select ID so it doesn't return everything else + select: { id: true }, + }), + ]); + userLog.success(`Imported general settings and ${settings.tags.length} tags`); + + userLog.info('Importing categories'); + const newCategories = await client.prisma.$transaction( + categories.map(category => { + delete category.id; + return client.prisma.category.create({ + data: { + ...category, + guild: { connect: { id } }, + questions: { + createMany: { + data: category.questions.map(question => { + delete question.categoryId; + return question; + }), + }, + }, + }, + select: { id: true }, + }); + }), + ); + + // settingsJSON.category because categories has been mutated (no id) + const categoryMap = new Map(settingsJSON.categories.map((cat, idx) => ([cat.id, newCategories[idx].id]))); + + for (const category of settingsJSON.categories) { + userLog.info(`"${category.name}" ID ${category.id} -> ${categoryMap.get(category.id)}`); + } + + userLog.success(`Imported ${categories.length} categories`); + + userLog.info('Reading tickets.jsonl'); + const stream = files.find(f => f.path === 'tickets.jsonl').stream(); + const lines = createInterface({ + crlfDelay: Infinity, + input: stream, + }); + const ticketsPromises = []; + + userLog.info('Encrypting tickets'); + + for await (const line of lines) { + // do not await in the loop + ticketsPromises.push(pool.queue(worker => worker.importTicket(line, id, categoryMap))); + } + + const ticketsResolved = await Promise.all(ticketsPromises); + const queries = []; + const allMessages = []; + + for (const [ticket, ticketMessages] of ticketsResolved) { + queries.push(client.prisma.ticket.create({ data: ticket })); + allMessages.push(...ticketMessages); + } + + if (allMessages.length > 0) { + queries.push(client.prisma.archivedMessage.createMany({ data: allMessages })); + } + + userLog.info('Importing tickets'); + await client.prisma.$transaction(queries); + userLog.success(`Imported ${ticketsResolved.length} tickets`); + userLog.success('(DONE) All data has been imported'); + + } catch (error) { + client.log.error(error); + userLog.error(error); + } finally { + res.raw.end(); + } + }, + onRequest: [fastify.authenticate, fastify.isAdmin], +}); diff --git a/src/routes/api/admin/guilds/[guild]/index.js b/src/routes/api/admin/guilds/[guild]/index.js index 0ae4538..59a68c7 100644 --- a/src/routes/api/admin/guilds/[guild]/index.js +++ b/src/routes/api/admin/guilds/[guild]/index.js @@ -12,8 +12,11 @@ module.exports.delete = fastify => ({ /** @type {import('client')} */ const client = req.routeOptions.config.client; const id = req.params.guild; - await client.prisma.guild.delete({ where: { id } }); - await client.prisma.guild.create({ data: { id } }); + client.keyv.delete(`cache/stats/guild:${id}`); + await client.prisma.$transaction([ + client.prisma.guild.delete({ where: { id } }), + client.prisma.guild.create({ data: { id } }), + ]); logAdminEvent(client, { action: 'delete', guildId: id, diff --git a/src/routes/api/client.js b/src/routes/api/client.js index 7efa8a6..cb92f3b 100644 --- a/src/routes/api/client.js +++ b/src/routes/api/client.js @@ -2,6 +2,7 @@ const { getAvgResolutionTime, getAvgResponseTime, } = require('../../lib/stats'); const ms = require('ms'); +const pkg = require('../../../package.json'); module.exports.get = () => ({ handler: async req => { @@ -20,6 +21,7 @@ module.exports.get = () => ({ }); const closedTickets = tickets.filter(t => t.firstResponseAt && t.closedAt); const users = await client.prisma.user.findMany({ select: { messageCount: true } }); + // TODO: background cached = { avatar: client.user.avatarURL(), discriminator: client.user.discriminator, @@ -37,6 +39,7 @@ module.exports.get = () => ({ tickets: tickets.length, }, username: client.user.username, + version: pkg.version, }; await client.keyv.set(cacheKey, cached, ms('15m')); }