Hessenuk/DiscordTickets
Hessenuk/DiscordTickets
2
0
Fork 0
mirror of https://github.com/Hessenuk/DiscordTickets.git synced 2025-02-23 18:51:29 +02:00
Code Issues Packages Projects Releases Wiki Activity

perf(api): faster exports

Browse Source
This commit is contained in:
Isaac 2025-02-01 01:54:59 +00:00
parent f029a49877
commit a33c670fc8
No known key found for this signature in database
GPG Key ID: 0DE40AE37BBA5C33
1 changed files with 6 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
src/routes/api/admin/guilds/[guild]/export.js
View File

@ -1,15 +1,15 @@
const { Readable } = require('node:stream');
const { cpus } = require('node:os');
const { const {
spawn, spawn,
Pool, Pool,
Worker, Worker,
} = require('threads'); } = require('threads');
const { Readable } = require('node:stream');
const { cpus } = require('node:os');
const archiver = require('archiver'); const archiver = require('archiver');
const { once } = require('node:events');
// ! ceiL: at least 1 // ! ceiL: at least 1
const poolSize = Math.ceil(cpus().length / 4); const poolSize = Math.ceil(cpus().length / 4);
const pool = Pool(() => spawn(new Worker('../../../../../lib/workers/export.js')), { size: poolSize });
module.exports.get = fastify => ({ module.exports.get = fastify => ({
/** /**
@ -47,7 +47,6 @@ module.exports.get = fastify => ({
const ticketsStream = Readable.from(ticketsGenerator()); const ticketsStream = Readable.from(ticketsGenerator());
async function* ticketsGenerator() { async function* ticketsGenerator() {
const pool = Pool(() => spawn(new Worker('../../../../../lib/workers/export.js')), { size: poolSize });
try { try {
let done = false; let done = false;
const findOptions = { const findOptions = {
@ -71,20 +70,14 @@ module.exports.get = fastify => ({
findOptions.skip = 1; findOptions.skip = 1;
findOptions.cursor = { id: batch[findOptions.take - 1].id }; findOptions.cursor = { id: batch[findOptions.take - 1].id };
} }
// ! map not for...of. // ! map (parallel) not for...of (serial)
// ! ! batch at a time, many tickets at a time per batch yield* batch.map(async ticket => (await pool.queue(worker => worker.exportTicket(ticket)) + '\n'));
const ar = await Promise.all(batch.map(async ticket => (await pool.queue(worker => worker.exportTicket(ticket)) + '\n')));
yield* ar;
} while (!done); } while (!done);
} finally { } finally {
await pool.terminate();
ticketsStream.push(null); // ! extremely important ticketsStream.push(null); // ! extremely important
} }
} }
const archive = archiver('zip', { const archive = archiver('zip', {
comment: JSON.stringify({ comment: JSON.stringify({
exportedAt: new Date().toISOString(), exportedAt: new Date().toISOString(),
@ -93,18 +86,15 @@ module.exports.get = fastify => ({
}) })
.append(JSON.stringify(settings), { name: 'settings.json' }) .append(JSON.stringify(settings), { name: 'settings.json' })
.append(ticketsStream, { name: 'tickets.jsonl' }); .append(ticketsStream, { name: 'tickets.jsonl' });
archive.finalize();
const cleanGuildName = guild.name.replace(/\W/g, '_').replace(/_+/g, '_'); const cleanGuildName = guild.name.replace(/\W/g, '_').replace(/_+/g, '_');
const fileName = `tickets-${cleanGuildName}-${new Date().toISOString().slice(0, 10)}`; const fileName = `tickets-${cleanGuildName}-${new Date().toISOString().slice(0, 10)}`;
res res
.type('application/zip') .type('application/zip')
.header('content-disposition', `attachment; filename="${fileName}"`) .header('content-disposition', `attachment; filename="${fileName}"`)
.send(archive); .send(archive);
await once(ticketsStream, 'end');
await archive.finalize();
}, },
onRequest: [fastify.authenticate, fastify.isAdmin], onRequest: [fastify.authenticate, fastify.isAdmin],
}); });