feat(api): data imports

src/lib/workers/export.js

@@ -2,34 +2,35 @@ const { expose } = require('threads/worker');
 const Cryptr = require('cryptr');
 const { decrypt } = new Cryptr(process.env.ENCRYPTION_KEY);
 
-function decryptIfExists(encrypted) {
-	if (encrypted) return decrypt(encrypted);
-	return null;
-}
-
 expose({
 	exportTicket(ticket) {
-		if (ticket.closedReason) ticket.closedReason = decrypt(ticket.closedReason);
-		if (ticket.feedback?.comment) ticket.feedback.comment = decrypt(ticket.feedback.comment);
-		if (ticket.topic) ticket.topic = decrypt(ticket.topic);
-
 		ticket.archivedMessages = ticket.archivedMessages.map(message => {
-			message.content = decryptIfExists(message.content);
+			message.content &&= decrypt(message.content);
 			return message;
 		});
 
 		ticket.archivedUsers = ticket.archivedUsers.map(user => {
-			user.displayName = decryptIfExists(user.displayName);
+			user.displayName &&= decrypt(user.displayName);
-			user.username = decryptIfExists(user.username);
+			user.username &&= decrypt(user.username);
 			return user;
 		});
 
+		if (ticket.feedback) {
+			// why is feedback the only one with a guild relation? 😕
+			delete ticket.feedback.guildId;
+			ticket.feedback.comment &&= decrypt(ticket.feedback.comment);
+		}
+
+		ticket.closedReason &&= decrypt(ticket.closedReason);
+
+		delete ticket.guildId;
+
 		ticket.questionAnswers = ticket.questionAnswers.map(answer => {
-			if (answer.value) answer.value = decryptIfExists(answer.value);
+			answer.value &&= decrypt(answer.value);
 			return answer;
 		});
 
-		delete ticket.guildId;
+		ticket.topic &&= decrypt(ticket.topic);
 
 		return JSON.stringify(ticket);
 	},

src/lib/workers/import.js

@@ -0,0 +1,115 @@
+const { expose } = require('threads/worker');
+const Cryptr = require('cryptr');
+const { encrypt } = new Cryptr(process.env.ENCRYPTION_KEY);
+
+expose({
+	importTicket(stringified, guildId, categoryMap) {
+		const ticket = JSON.parse(stringified);
+
+		ticket.archivedChannels = {
+			create: ticket.archivedChannels.map(user => {
+				delete user.ticketId;
+				return user;
+			}),
+		};
+
+		ticket.archivedUsers = {
+			create: ticket.archivedUsers.map(user => {
+				delete user.ticketId;
+				user.displayName &&= encrypt(user.displayName);
+				user.username &&= encrypt(user.username);
+				return user;
+			}),
+		};
+
+		ticket.archivedRoles = {
+			create: ticket.archivedRoles.map(user => {
+				delete user.ticketId;
+				return user;
+			}),
+		};
+
+		const messages = ticket.archivedMessages.map(message => {
+			// messages don't need to be wrapped in {create}
+			message.content &&= encrypt(message.content);
+			return message;
+		});
+		delete ticket.archivedMessages;
+
+		ticket.category = { connect: { id: categoryMap.get(ticket.categoryId) } };
+		delete ticket.categoryId;
+
+		if (ticket.claimedById) {
+			ticket.claimedBy = {
+				connectOrCreate: {
+					create: { id: ticket.claimedById },
+					where: { id: ticket.claimedById },
+				},
+			};
+		}
+		delete ticket.claimedById;
+
+		if (ticket.closedById) {
+			ticket.closedBy = {
+				connectOrCreate: {
+					create: { id: ticket.closedById },
+					where: { id: ticket.closedById },
+				},
+			};
+		}
+		delete ticket.closedById;
+
+		if (ticket.createdById) {
+			ticket.createdBy = {
+				connectOrCreate: {
+					create: { id: ticket.createdById },
+					where: { id: ticket.createdById },
+				},
+			};
+		}
+		delete ticket.createdById;
+
+		ticket.closedReason &&= encrypt(ticket.closedReason);
+
+		if (ticket.feedback) {
+			ticket.feedback.guild = { connect: { id: guildId } };
+			ticket.feedback.comment &&= encrypt(ticket.feedback.comment);
+			if (ticket.feedback.userId) {
+				ticket.feedback.user = {
+					connectOrCreate: {
+						create: { id: ticket.feedback.userId },
+						where: { id: ticket.feedback.userId },
+					},
+				};
+				delete ticket.feedback.userId;
+			}
+			ticket.feedback = { create: ticket.feedback };
+		} else {
+			ticket.feedback = undefined;
+		}
+
+		ticket.guild = { connect: { id: guildId } };
+		delete ticket.guildId; // shouldn't exist but make sure
+
+		if (ticket.questionAnswers.length) {
+			ticket.questionAnswers = {
+				create: ticket.questionAnswers.map(async answer => {
+					answer.value &&= encrypt(answer.value);
+					return answer;
+				}),
+			};
+		} else {
+			ticket.questionAnswers = undefined;
+		}
+
+		if (ticket.referencesTicketId) {
+			ticket.referencesTicket = { connect: { id: ticket.referencesTicketId } };
+		}
+		delete ticket.referencesTicketId;
+
+		ticket.topic &&= encrypt(ticket.topic);
+
+		return [ticket, messages];
+
+	},
+});

src/routes/api/admin/guilds/[guild]/export.js

@@ -6,6 +6,8 @@ const {
 const { Readable } = require('node:stream');
 const { cpus } = require('node:os');
 const archiver = require('archiver');
+const { iconURL } = require('../../../../../lib/misc');
+const pkg = require('../../../../../../package.json');
 
 // ! ceiL: at least 1
 const poolSize = Math.ceil(cpus().length / 4);
@@ -26,6 +28,26 @@ module.exports.get = fastify => ({
 
 		client.log.info(`${member.user.username} requested an export of "${guild.name}"`);
 
+		// TODO: sign so the importer can ensure files haven't been added (important for attachments)
+		const archive = archiver('zip', {
+			comment: JSON.stringify({
+				exportedAt: new Date().toISOString(),
+				exportedFromClientId: client.user.id,
+				originalGuildId: id,
+				originalGuildName: guild.name,
+				version: pkg.version,
+			}),
+		});
+
+		archive.on('warning', err => {
+			if (err.code === 'ENOENT') client.log.warn(err);
+			else throw err;
+		});
+
+		archive.on('error', err => {
+			throw err;
+		});
+
 		const settings = await client.prisma.guild.findUnique({
 			include: {
 				categories: { include: { questions: true } },
@@ -34,6 +56,8 @@ module.exports.get = fastify => ({
 			where: { id },
 		});
 
+		delete settings.id;
+
 		settings.categories = settings.categories.map(c => {
 			delete c.guildId;
 			return c;
@@ -45,7 +69,6 @@ module.exports.get = fastify => ({
 		});
 
 		const ticketsStream = Readable.from(ticketsGenerator());
-
 		async function* ticketsGenerator() {
 			try {
 				let done = false;
@@ -72,21 +95,23 @@ module.exports.get = fastify => ({
 					}
 					// ! map (parallel) not for...of (serial)
 					yield* batch.map(async ticket => (await pool.queue(worker => worker.exportTicket(ticket)) + '\n'));
+					// Readable.from(AsyncGenerator) seems to be faster than pushing to a Readable with an empty `read()` function
+					// for (const ticket of batch) {
+					// 	pool
+					// 		.queue(worker => worker.exportTicket(ticket))
+					// 		.then(string => ticketsStream.push(string + '\n'));
+					// }
 				} while (!done);
 			} finally {
 				ticketsStream.push(null); // ! extremely important
 			}
 		}
 
-		const archive = archiver('zip', {
+		const icon = await fetch(iconURL(guild));
-			comment: JSON.stringify({
+		archive.append(Readable.from(icon.body), { name: 'icon.png' });
-				exportedAt: new Date().toISOString(),
+		archive.append(JSON.stringify(settings), { name: 'settings.json' });
-				originalGuildId: id,
+		archive.append(ticketsStream, { name: 'tickets.jsonl' });
-			}),
+		archive.finalize(); // ! do not await
-		})
-			.append(JSON.stringify(settings), { name: 'settings.json' })
-			.append(ticketsStream, { name: 'tickets.jsonl' });
-		archive.finalize();
 
 		const cleanGuildName = guild.name.replace(/\W/g, '_').replace(/_+/g, '_');
 		const fileName = `tickets-${cleanGuildName}-${new Date().toISOString().slice(0, 10)}`;

src/routes/api/admin/guilds/[guild]/import.js

@@ -0,0 +1,186 @@
+
+const {
+	spawn,
+	Pool,
+	Worker,
+} = require('threads');
+const { cpus } = require('node:os');
+const unzipper = require('unzipper');
+const { createInterface } = require('node:readline');
+const pkg = require('../../../../../../package.json');
+
+// ! ceiL: at least 1
+const poolSize = Math.ceil(cpus().length / 4);
+const pool = Pool(() => spawn(new Worker('../../../../../lib/workers/import.js')), { size: poolSize });
+
+function parseJSON(string) {
+	try {
+		return JSON.parse(string);
+	} catch {
+		return null;
+	}
+}
+
+// put would be better but forms can only get or post
+module.exports.post = fastify => ({
+	/**
+	 *
+	 * @param {import('fastify').FastifyRequest} req
+	 * @param {import('fastify').FastifyReply} res
+	 */
+	handler: async (req, res) => {
+		/** @type {import('client')} */
+		const client = req.routeOptions.config.client;
+		const id = req.params.guild;
+		const guild = client.guilds.cache.get(id);
+		const member = await guild.members.fetch(req.user.id);
+
+		client.log.info(`${member.user.username} is importing data to "${guild.name}"`);
+
+		client.keyv.delete(`cache/stats/guild:${id}`);
+
+		const [zFile] = await req.saveRequestFiles({
+			limits: {
+				fields: 1,
+				files: 1,
+			},
+		});
+
+		res.raw.writeHead(200, { 'Content-Type': 'text/html; charset=utf-8' });
+
+		const userLog = {
+			_write(style1, style2, prefix, string) {
+				res.raw.write(`<p><span class="${style1}">[${prefix}]</span> <span class="${style2}">${string}</span></p>`);
+			},
+			error(string) {
+				this._write('text-red-500 font-bold', 'text-red-700 dark:text-red-200', 'ERROR', string);
+			},
+			info(string) {
+				this._write('text-cyan-500', 'text-cyan-700 dark:text-cyan-200', 'INFO', string);
+			},
+			success(string) {
+				this._write('text-green-500', 'text-green-700 dark:text-green-200', 'SUCCESS', string);
+			},
+			warn(string) {
+				this._write('text-orange-500', 'text-orange-700 dark:text-orange-200', 'WARN', string);
+			},
+		};
+
+		try {
+		// comment needs to be less than 512B
+			const zip = await unzipper.Open.file(zFile.filepath, { tailSize: 512 });
+			const { files } = zip;
+			const comment = parseJSON(zip.comment);
+			client.log.info('Import comment', comment);
+			if (comment) {
+				userLog.info(`v${comment.version} -> v${pkg.version}`);
+			} else {
+				userLog.warn('Comment is not parsable');
+			}
+
+			userLog.info('Reading settings.json');
+			// `settingsJSON` is frozen, `settings` can be mutated
+			const settingsJSON = JSON.parse(await files.find(f => f.path === 'settings.json').buffer());
+			Object.freeze(settingsJSON);
+			const settings = structuredClone(settingsJSON);
+			const { categories } = settings;
+			delete settings.categories; // this also mutates `settings
+
+			userLog.info('Importing general settings and tags');
+			await client.prisma.$transaction([
+				client.prisma.guild.delete({
+					select: { id: true },
+					where: { id },
+				}),
+				client.prisma.guild.create({
+					data: {
+						...settings,
+						id,
+						tags: {
+							createMany: {
+								data: settings.tags.map(tag => {
+									delete tag.id;
+									return tag;
+								}),
+							},
+						},
+					},
+					// select ID so it doesn't return everything else
+					select: { id: true },
+				}),
+			]);
+			userLog.success(`Imported general settings and ${settings.tags.length} tags`);
+
+			userLog.info('Importing categories');
+			const newCategories = await client.prisma.$transaction(
+				categories.map(category => {
+					delete category.id;
+					return client.prisma.category.create({
+						data: {
+							...category,
+							guild: { connect: { id } },
+							questions: {
+								createMany: {
+									data: category.questions.map(question => {
+										delete question.categoryId;
+										return question;
+									}),
+								},
+							},
+						},
+						select: { id: true },
+					});
+				}),
+			);
+
+			// settingsJSON.category because categories has been mutated (no id)
+			const categoryMap = new Map(settingsJSON.categories.map((cat, idx) => ([cat.id, newCategories[idx].id])));
+
+			for (const category of settingsJSON.categories) {
+				userLog.info(`"${category.name}" ID ${category.id} -> ${categoryMap.get(category.id)}`);
+			}
+
+			userLog.success(`Imported ${categories.length} categories`);
+
+			userLog.info('Reading tickets.jsonl');
+			const stream = files.find(f => f.path === 'tickets.jsonl').stream();
+			const lines = createInterface({
+				crlfDelay: Infinity,
+				input: stream,
+			});
+			const ticketsPromises = [];
+
+			userLog.info('Encrypting tickets');
+
+			for await (const line of lines) {
+				// do not await in the loop
+				ticketsPromises.push(pool.queue(worker => worker.importTicket(line, id, categoryMap)));
+			}
+
+			const ticketsResolved = await Promise.all(ticketsPromises);
+			const queries = [];
+			const allMessages = [];
+
+			for (const [ticket, ticketMessages] of ticketsResolved) {
+				queries.push(client.prisma.ticket.create({ data: ticket }));
+				allMessages.push(...ticketMessages);
+			}
+
+			if (allMessages.length > 0) {
+				queries.push(client.prisma.archivedMessage.createMany({ data: allMessages }));
+			}
+
+			userLog.info('Importing tickets');
+			await client.prisma.$transaction(queries);
+			userLog.success(`Imported ${ticketsResolved.length} tickets`);
+			userLog.success('(DONE) All data has been imported');
+
+		} catch (error) {
+			client.log.error(error);
+			userLog.error(error);
+		} finally {
+			res.raw.end();
+		}
+	},
+	onRequest: [fastify.authenticate, fastify.isAdmin],
+});

src/routes/api/admin/guilds/[guild]/index.js

@@ -12,8 +12,11 @@ module.exports.delete = fastify => ({
 		/** @type {import('client')} */
 		const client = req.routeOptions.config.client;
 		const id = req.params.guild;
-		await client.prisma.guild.delete({ where: { id } });
+		client.keyv.delete(`cache/stats/guild:${id}`);
-		await client.prisma.guild.create({ data: { id } });
+		await client.prisma.$transaction([
+			client.prisma.guild.delete({ where: { id } }),
+			client.prisma.guild.create({ data: { id } }),
+		]);
 		logAdminEvent(client, {
 			action: 'delete',
 			guildId: id,

src/routes/api/client.js

@@ -2,6 +2,7 @@ const {
 	getAvgResolutionTime, getAvgResponseTime,
 } = require('../../lib/stats');
 const ms = require('ms');
+const pkg = require('../../../package.json');
 
 module.exports.get = () => ({
 	handler: async req => {
@@ -20,6 +21,7 @@ module.exports.get = () => ({
 			});
 			const closedTickets = tickets.filter(t => t.firstResponseAt && t.closedAt);
 			const users = await client.prisma.user.findMany({ select: { messageCount: true } });
+			// TODO: background
 			cached = {
 				avatar: client.user.avatarURL(),
 				discriminator: client.user.discriminator,
@@ -37,6 +39,7 @@ module.exports.get = () => ({
 					tickets: tickets.length,
 				},
 				username: client.user.username,
+				version: pkg.version,
 			};
 			await client.keyv.set(cacheKey, cached, ms('15m'));
 		}