diff --git a/src/routes/api/admin/guilds/[guild]/export.js b/src/routes/api/admin/guilds/[guild]/export.js index 89658f1..53934fe 100644 --- a/src/routes/api/admin/guilds/[guild]/export.js +++ b/src/routes/api/admin/guilds/[guild]/export.js @@ -9,8 +9,8 @@ const archiver = require('archiver'); const { iconURL } = require('../../../../../lib/misc'); const pkg = require('../../../../../../package.json'); -// ! ceiL: at least 1 -const poolSize = Math.ceil(cpus().length / 4); +// a single persistent pool shared across all exports +const poolSize = Math.ceil(cpus().length / 4); // ! ceiL: at least 1 const pool = Pool(() => spawn(new Worker('../../../../../lib/workers/export.js')), { size: poolSize }); module.exports.get = fastify => ({ @@ -72,6 +72,7 @@ module.exports.get = fastify => ({ async function* ticketsGenerator() { try { let done = false; + const take = 50; const findOptions = { include: { archivedChannels: true, @@ -82,16 +83,16 @@ module.exports.get = fastify => ({ questionAnswers: true, }, orderBy: { id: 'asc' }, - take: 24, + take, where: { guildId: id }, }; do { const batch = await client.prisma.ticket.findMany(findOptions); - if (batch.length < findOptions.take) { + if (batch.length < take) { done = true; } else { findOptions.skip = 1; - findOptions.cursor = { id: batch[findOptions.take - 1].id }; + findOptions.cursor = { id: batch[take - 1].id }; } // ! map (parallel) not for...of (serial) yield* batch.map(async ticket => (await pool.queue(worker => worker.exportTicket(ticket)) + '\n')); diff --git a/src/routes/api/admin/guilds/[guild]/import.js b/src/routes/api/admin/guilds/[guild]/import.js index a1d8db8..e63f637 100644 --- a/src/routes/api/admin/guilds/[guild]/import.js +++ b/src/routes/api/admin/guilds/[guild]/import.js @@ -9,8 +9,8 @@ const unzipper = require('unzipper'); const { createInterface } = require('node:readline'); const pkg = require('../../../../../../package.json'); -// ! ceiL: at least 1 -const poolSize = Math.ceil(cpus().length / 4); +// a single persistent pool shared across all imports +const poolSize = Math.ceil(cpus().length / 4); // ! ceiL: at least 1 const pool = Pool(() => spawn(new Worker('../../../../../lib/workers/import.js')), { size: poolSize }); function parseJSON(string) { @@ -157,6 +157,7 @@ module.exports.post = fastify => ({ ticketsPromises.push(pool.queue(worker => worker.importTicket(line, id, categoryMap))); } + // TODO: batch 100 tickets per query? const ticketsResolved = await Promise.all(ticketsPromises); const queries = []; const allMessages = [];