Hessenuk/DiscordTickets
Hessenuk/DiscordTickets
2
0
Fork 0
mirror of https://github.com/Hessenuk/DiscordTickets.git synced 2025-02-23 10:51:22 +02:00
Code Issues Packages Projects Releases Wiki Activity

perf(api): increase batch size

Browse Source
This commit is contained in:
Isaac 2025-02-12 16:20:13 +00:00
parent 33c1b64019
commit a861f76df9
No known key found for this signature in database
GPG Key ID: 0DE40AE37BBA5C33
2 changed files with 9 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
src/routes/api/admin/guilds/[guild]/export.js
View File

@ -9,8 +9,8 @@ const archiver = require('archiver');
const { iconURL } = require('../../../../../lib/misc');
const pkg = require('../../../../../../package.json');
// ! ceiL: at least 1
const poolSize = Math.ceil(cpus().length / 4);
// a single persistent pool shared across all exports
const poolSize = Math.ceil(cpus().length / 4); // ! ceiL: at least 1
const pool = Pool(() => spawn(new Worker('../../../../../lib/workers/export.js')), { size: poolSize });
module.exports.get = fastify => ({
@ -72,6 +72,7 @@ module.exports.get = fastify => ({
async function* ticketsGenerator() {
try {
let done = false;
const take = 50;
const findOptions = {
include: {
archivedChannels: true,
@ -82,16 +83,16 @@ module.exports.get = fastify => ({
questionAnswers: true,
},
orderBy: { id: 'asc' },
take: 24,
take,
where: { guildId: id },
};
do {
const batch = await client.prisma.ticket.findMany(findOptions);
if (batch.length < findOptions.take) {
if (batch.length < take) {
done = true;
} else {
findOptions.skip = 1;
findOptions.cursor = { id: batch[findOptions.take - 1].id };
findOptions.cursor = { id: batch[take - 1].id };
}
// ! map (parallel) not for...of (serial)
yield* batch.map(async ticket => (await pool.queue(worker => worker.exportTicket(ticket)) + '\n'));

5
src/routes/api/admin/guilds/[guild]/import.js
View File

@ -9,8 +9,8 @@ const unzipper = require('unzipper');
const { createInterface } = require('node:readline');
const pkg = require('../../../../../../package.json');
// ! ceiL: at least 1
const poolSize = Math.ceil(cpus().length / 4);
// a single persistent pool shared across all imports
const poolSize = Math.ceil(cpus().length / 4); // ! ceiL: at least 1
const pool = Pool(() => spawn(new Worker('../../../../../lib/workers/import.js')), { size: poolSize });
function parseJSON(string) {
@ -157,6 +157,7 @@ module.exports.post = fastify => ({
ticketsPromises.push(pool.queue(worker => worker.importTicket(line, id, categoryMap)));
}
// TODO: batch 100 tickets per query?
const ticketsResolved = await Promise.all(ticketsPromises);
const queries = [];
const allMessages = [];