From 6b0146e0997612602c0f161195167f090dc15a6f Mon Sep 17 00:00:00 2001 From: Isaac Date: Wed, 12 Feb 2025 01:37:44 +0000 Subject: [PATCH] perf(stats): threads, better & parallel queries --- src/lib/stats.js | 52 +++++++------- src/lib/threads.js | 59 ++++++++++++++++ src/lib/tickets/manager.js | 13 ++-- src/lib/workers/stats.js | 9 ++- src/listeners/client/ready.js | 19 ++--- .../admin/guilds/[guild]/categories/index.js | 47 ++++++++----- src/routes/api/admin/guilds/[guild]/index.js | 69 ++++++++++++------- src/routes/api/client.js | 68 +++++++++++------- 8 files changed, 230 insertions(+), 106 deletions(-) create mode 100644 src/lib/threads.js diff --git a/src/lib/stats.js b/src/lib/stats.js index fecb967..d12bb99 100644 --- a/src/lib/stats.js +++ b/src/lib/stats.js @@ -1,27 +1,22 @@ /* eslint-disable no-underscore-dangle */ -const { - spawn, - Pool, - Worker, -} = require('threads'); -const { cpus } = require('node:os'); const { version } = require('../../package.json'); const { md5 } = require('./misc'); +const { + quick, + relativePool, +} = require('./threads'); -// ! ceiL: at least 1 -const poolSize = Math.ceil(cpus().length / 4); -const pool = Pool(() => spawn(new Worker('./workers/stats.js')), { size: poolSize }); - -module.exports.getAvgResolutionTime = tickets => (tickets.reduce((total, ticket) => total + (ticket.closedAt - ticket.createdAt), 0) || 1) / Math.max(tickets.length, 1); - -module.exports.getAvgResponseTime = tickets => (tickets.reduce((total, ticket) => total + (ticket.firstResponseAt - ticket.createdAt), 0) || 1) / Math.max(tickets.length, 1); +const getAverageTimes = closedTickets => quick('stats', async w => ({ + avgResolutionTime: await w.getAvgResolutionTime(closedTickets), + avgResponseTime: await w.getAvgResponseTime(closedTickets), +})); /** - * + * Report stats to Houston * @param {import("../client")} client */ -module.exports.sendToHouston = async client => { +async function sendToHouston(client) { const guilds = await client.prisma.guild.findMany({ include: { categories: { include: { _count: { select: { questions: true } } } }, @@ -35,30 +30,32 @@ module.exports.sendToHouston = async client => { }, }, }); - const users = (await client.prisma.user.aggregate({ + const users = await client.prisma.user.aggregate({ _count: true, _sum: { messageCount: true }, - })); - const messages = users._sum.messageCount ?? 0; + }); + const messages = users._sum.messageCount; const stats = { activated_users: users._count, arch: process.arch, database: process.env.DB_PROVIDER, guilds: await Promise.all( - guilds - .filter(guild => client.guilds.cache.has(guild.id)) - .map(async guild => { - guild.members = client.guilds.cache.get(guild.id).memberCount; - return pool.queue(worker => worker.aggregateGuildForHouston(guild, messages)); - }), + await relativePool(0.25, 'stats', pool => + guilds + .filter(guild => client.guilds.cache.has(guild.id)) + .map(async guild => { + guild.members = client.guilds.cache.get(guild.id).memberCount; + return pool.queue(w => w.aggregateGuildForHouston(guild, messages)); + }), + ), ), id: md5(client.user.id), node: process.version, os: process.platform, version, }; - const delta = guilds.length - stats.guilds.length; + if (delta !== 0) { client.log.warn('%d guilds are not cached and were excluded from the stats report', delta); } @@ -84,3 +81,8 @@ module.exports.sendToHouston = async client => { client.log.debug(res); } }; + +module.exports = { + getAverageTimes, + sendToHouston, +}; diff --git a/src/lib/threads.js b/src/lib/threads.js new file mode 100644 index 0000000..d351ad7 --- /dev/null +++ b/src/lib/threads.js @@ -0,0 +1,59 @@ +const { + spawn, + Pool, + Thread, + Worker, +} = require('threads'); +const { cpus } = require('node:os'); + +/** + * Use a thread pool of a fixed size + * @param {number} size number of threads + * @param {string} name name of file in workers directory + * @param {function} fun async function + * @returns {Promise} + */ +async function pool(size, name, fun) { + const pool = Pool(() => spawn(new Worker(`./workers/${name}.js`)), { size }); + try { + return await fun(pool); + } finally { + await pool.settled(); + await pool.terminate(); + } +}; + +/** + * Spawn one thread, do something, and terminate it + * @param {string} name name of file in workers directory + * @param {function} fun async function + * @returns {Promise} + */ +function relativePool(fraction, ...args) { + // ! ceiL: at least 1 + const poolSize = Math.ceil(fraction * cpus().length); + return pool(poolSize, ...args); +} + +module.exports = { + pool, + quick, + relativePool, +}; diff --git a/src/lib/tickets/manager.js b/src/lib/tickets/manager.js index fe02f33..d8ed05d 100644 --- a/src/lib/tickets/manager.js +++ b/src/lib/tickets/manager.js @@ -20,15 +20,12 @@ const { isStaff } = require('../users'); const { Collection } = require('discord.js'); const spacetime = require('spacetime'); const Cryptr = require('cryptr'); -const { - getAvgResolutionTime, - getAvgResponseTime, -} = require('../stats'); const { decrypt, encrypt, } = new Cryptr(process.env.ENCRYPTION_KEY); const { getSUID } = require('../logging'); +const { getAverageTimes } = require('../stats'); /** * @typedef {import('@prisma/client').Category & @@ -434,9 +431,13 @@ module.exports = class TicketManager { open: false, }, }); + const { + avgResolutionTime, + avgResponseTime, + } = await getAverageTimes(closedTickets); stats = { - avgResolutionTime: ms(getAvgResolutionTime(closedTickets), { long: true }), - avgResponseTime: ms(getAvgResponseTime(closedTickets), { long: true }), + avgResolutionTime: ms(avgResolutionTime, { long: true }), + avgResponseTime: ms(avgResponseTime, { long: true }), }; this.client.keyv.set(statsCacheKey, stats, ms('1h')); } diff --git a/src/lib/workers/stats.js b/src/lib/workers/stats.js index 7fb929d..4ffdf63 100644 --- a/src/lib/workers/stats.js +++ b/src/lib/workers/stats.js @@ -7,9 +7,13 @@ const md5 = str => createHash('md5').update(str).digest('hex'); const msToMins = ms => Number((ms / 1000 / 60).toFixed(2)); -const getAvgResolutionTime = tickets => (tickets.reduce((total, ticket) => total + (ticket.closedAt - ticket.createdAt), 0) || 1) / Math.max(tickets.length, 1); +const reduce = (closedTickets, prop) => closedTickets.reduce((total, ticket) => total + (ticket[prop] - ticket.createdAt), 0) || 1; -const getAvgResponseTime = tickets => (tickets.reduce((total, ticket) => total + (ticket.firstResponseAt - ticket.createdAt), 0) || 1) / Math.max(tickets.length, 1); +const getAvgResolutionTime = closedTickets => reduce(closedTickets, 'closedAt') / Math.max(closedTickets.length, 1); + +const getAvgResponseTime = closedTickets => reduce(closedTickets, 'firstResponseAt') / Math.max(closedTickets.length, 1); + +const sum = numbers => numbers.reduce((t, n) => t + n, 0); expose({ aggregateGuildForHouston(guild, messages) { @@ -37,4 +41,5 @@ expose({ }, getAvgResolutionTime, getAvgResponseTime, + sum, }); diff --git a/src/listeners/client/ready.js b/src/listeners/client/ready.js index 19bd2d9..71b7e4f 100644 --- a/src/listeners/client/ready.js +++ b/src/listeners/client/ready.js @@ -1,8 +1,4 @@ const { Listener } = require('@eartharoid/dbf'); -const { - getAvgResolutionTime, - getAvgResponseTime, -} = require('../../lib/stats'); const ms = require('ms'); const sync = require('../../lib/sync'); const checkForUpdates = require('../../lib/updates'); @@ -13,7 +9,10 @@ const { ButtonStyle, } = require('discord.js'); const ExtendedEmbedBuilder = require('../../lib/embed'); -const { sendToHouston } = require('../../lib/stats'); +const { + getAverageTimes, + sendToHouston, +} = require('../../lib/stats'); module.exports = class extends Listener { constructor(client, options) { @@ -69,11 +68,15 @@ module.exports = class extends Listener { firstResponseAt: true, }, }); - const closedTicketsWithResponse = tickets.filter(t => t.firstResponseAt && t.closedAt); const closedTickets = tickets.filter(t => t.closedAt); + const closedTicketsWithResponse = closedTickets.filter(t => t.firstResponseAt); + const { + avgResolutionTime, + avgResponseTime, + } = await getAverageTimes(closedTicketsWithResponse); cached = { - avgResolutionTime: ms(getAvgResolutionTime(closedTicketsWithResponse)), - avgResponseTime: ms(getAvgResponseTime(closedTicketsWithResponse)), + avgResolutionTime: ms(avgResolutionTime), + avgResponseTime: ms(avgResponseTime), guilds: client.guilds.cache.size, openTickets: tickets.length - closedTickets.length, totalTickets: tickets.length, diff --git a/src/routes/api/admin/guilds/[guild]/categories/index.js b/src/routes/api/admin/guilds/[guild]/categories/index.js index d8da7d7..9f329db 100644 --- a/src/routes/api/admin/guilds/[guild]/categories/index.js +++ b/src/routes/api/admin/guilds/[guild]/categories/index.js @@ -6,10 +6,7 @@ const { ChannelType: { GuildCategory }, } = require('discord.js'); const ms = require('ms'); -const { - getAvgResolutionTime, - getAvgResponseTime, -} = require('../../../../../../lib/stats'); +const { getAverageTimes } = require('../../../../../../lib/stats'); module.exports.get = fastify => ({ handler: async req => { @@ -29,24 +26,40 @@ module.exports.get = fastify => ({ name: true, requiredRoles: true, staffRoles: true, - tickets: { where: { open: false } }, + tickets: { + select: { + closedAt: true, + createdAt: true, + firstResponseAt: true, + }, + where: { + firstResponseAt: { not: null }, + open: false, + }, + }, }, }, }, where: { id: req.params.guild }, }); - categories = categories.map(c => { - const closedTickets = c.tickets.filter(t => t.firstResponseAt && t.closedAt); - c = { - ...c, - stats: { - avgResolutionTime: ms(getAvgResolutionTime(closedTickets)), - avgResponseTime: ms(getAvgResponseTime(closedTickets)), - }, - }; - delete c.tickets; - return c; - }); + + categories = await Promise.all( + categories.map(async category => { + const { + avgResolutionTime, + avgResponseTime, + } = await getAverageTimes(category.tickets); + category = { + ...category, + stats: { + avgResolutionTime: ms(avgResolutionTime), + avgResponseTime: ms(avgResponseTime), + }, + }; + delete category.tickets; + return category; + }), + ); return categories; }, diff --git a/src/routes/api/admin/guilds/[guild]/index.js b/src/routes/api/admin/guilds/[guild]/index.js index 59a68c7..6b6c3ca 100644 --- a/src/routes/api/admin/guilds/[guild]/index.js +++ b/src/routes/api/admin/guilds/[guild]/index.js @@ -1,10 +1,7 @@ /* eslint-disable no-underscore-dangle */ const { logAdminEvent } = require('../../../../../lib/logging.js'); const { iconURL } = require('../../../../../lib/misc'); -const { - getAvgResolutionTime, - getAvgResponseTime, -} = require('../../../../../lib/stats'); +const { getAverageTimes } = require('../../../../../lib/stats'); const ms = require('ms'); module.exports.delete = fastify => ({ @@ -42,39 +39,61 @@ module.exports.get = fastify => ({ if (!cached) { const guild = client.guilds.cache.get(id); - const settings = await client.prisma.guild.findUnique({ where: { id } }) ?? + const settings = + await client.prisma.guild.findUnique({ where: { id } }) ?? await client.prisma.guild.create({ data: { id } }); - const categories = await client.prisma.category.findMany({ - select: { - _count: { select: { tickets: true } }, - id: true, - name: true, - }, - where: { guildId: id }, - }); - const tickets = await client.prisma.ticket.findMany({ - select: { - closedAt: true, - createdAt: true, - firstResponseAt: true, - }, - where: { guildId: id }, - }); - const closedTickets = tickets.filter(t => t.firstResponseAt && t.closedAt); + const [ + categories, + tags, + tickets, + closedTickets, + ] = await Promise.all([ + client.prisma.category.findMany({ + select: { + _count: { select: { tickets: true } }, + id: true, + name: true, + }, + where: { guildId: id }, + }), + client.prisma.tag.count({ where: { guildId: id } }), + client.prisma.ticket.count(), + client.prisma.ticket.findMany({ + select: { + closedAt: true, + createdAt: true, + firstResponseAt: true, + }, + where: { + firstResponseAt: { not: null }, + guildId: id, + open: false, + }, + }), + client.prisma.user.aggregate({ + _count: true, + _sum: { messageCount: true }, + }), + ]); + const { + avgResolutionTime, + avgResponseTime, + } = await getAverageTimes(closedTickets); + cached = { createdAt: settings.createdAt, id: guild.id, logo: iconURL(guild), name: guild.name, stats: { - avgResolutionTime: ms(getAvgResolutionTime(closedTickets)), - avgResponseTime: ms(getAvgResponseTime(closedTickets)), + avgResolutionTime: ms(avgResolutionTime), + avgResponseTime: ms(avgResponseTime), categories: categories.map(c => ({ id: c.id, name: c.name, tickets: c._count.tickets, })), - tags: await client.prisma.tag.count({ where: { guildId: id } }), + tags, tickets: tickets.length, }, }; diff --git a/src/routes/api/client.js b/src/routes/api/client.js index cb92f3b..824abb8 100644 --- a/src/routes/api/client.js +++ b/src/routes/api/client.js @@ -1,8 +1,9 @@ -const { - getAvgResolutionTime, getAvgResponseTime, -} = require('../../lib/stats'); +/* eslint-disable no-underscore-dangle */ + const ms = require('ms'); const pkg = require('../../../package.json'); +const { getAverageTimes } = require('../../lib/stats'); +const { quick } = require('../../lib/threads'); module.exports.get = () => ({ handler: async req => { @@ -10,40 +11,61 @@ module.exports.get = () => ({ const client = req.routeOptions.config.client; const cacheKey = 'cache/stats/client'; let cached = await client.keyv.get(cacheKey); - if (!cached) { - const tickets = await client.prisma.ticket.findMany({ - select: { - closedAt: true, - createdAt: true, - firstResponseAt: true, - }, - }); - const closedTickets = tickets.filter(t => t.firstResponseAt && t.closedAt); - const users = await client.prisma.user.findMany({ select: { messageCount: true } }); - // TODO: background + const [ + categories, + members, + tags, + tickets, + closedTickets, + users, + ] = await Promise.all([ + client.prisma.category.count(), + quick('stats', w => w.sum(client.guilds.cache.map(g => g.memberCount))), + client.prisma.tag.count(), + client.prisma.ticket.count(), + client.prisma.ticket.findMany({ + select: { + closedAt: true, + createdAt: true, + firstResponseAt: true, + }, + where: { + firstResponseAt: { not: null }, + open: false, + }, + }), + client.prisma.user.aggregate({ + _count: true, + _sum: { messageCount: true }, + }), + ]); + const { + avgResolutionTime, + avgResponseTime, + } = await getAverageTimes(closedTickets); + cached = { avatar: client.user.avatarURL(), discriminator: client.user.discriminator, id: client.user.id, public: (process.env.PUBLIC_BOT === 'true'), stats: { - activatedUsers: users.length, - archivedMessages: users.reduce((total, user) => total + user.messageCount, 0), // don't count archivedMessage table rows, they can be deleted - avgResolutionTime: ms(getAvgResolutionTime(closedTickets)), - avgResponseTime: ms(getAvgResponseTime(closedTickets)), - categories: await client.prisma.category.count(), + activatedUsers: users._count, + archivedMessages: users._sum.messageCount, + avgResolutionTime: ms(avgResolutionTime), + avgResponseTime: ms(avgResponseTime), + categories, guilds: client.guilds.cache.size, - members: client.guilds.cache.reduce((t, g) => t + g.memberCount, 0), - tags: await client.prisma.tag.count(), - tickets: tickets.length, + members, + tags, + tickets, }, username: client.user.username, version: pkg.version, }; await client.keyv.set(cacheKey, cached, ms('15m')); } - return cached; }, });