From 413bae6d2c1b74e7181e25d4a47d34bfb7e8baa3 Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 13 Feb 2025 01:10:37 +0000 Subject: [PATCH] feat: super fast database dump and restore scripts --- package.json | 4 +- scripts/dump.mjs | 61 +++++++++++++ scripts/export.mjs | 117 ------------------------- scripts/import.mjs | 208 -------------------------------------------- scripts/restore.mjs | 51 +++++++++++ 5 files changed, 114 insertions(+), 327 deletions(-) create mode 100644 scripts/dump.mjs delete mode 100644 scripts/export.mjs delete mode 100644 scripts/import.mjs create mode 100644 scripts/restore.mjs diff --git a/package.json b/package.json index a496524..f9b6f50 100644 --- a/package.json +++ b/package.json @@ -5,8 +5,8 @@ "description": "The most popular open-source ticket management bot for Discord.", "main": "src/", "scripts": { - "db.dump": "node --no-warnings scripts/dump.mjs", - "db.restore": "node --no-warnings scripts/restore.mjs", + "db.dump": "node scripts/dump.mjs", + "db.restore": "node scripts/restore.mjs", "changelog": "conventional-changelog -p angular -i CHANGELOG.md -s", "contributors:add": "all-contributors add", "contributors:generate": "all-contributors generate", diff --git a/scripts/dump.mjs b/scripts/dump.mjs new file mode 100644 index 0000000..9c42491 --- /dev/null +++ b/scripts/dump.mjs @@ -0,0 +1,61 @@ +import { config } from 'dotenv'; +import fse from 'fs-extra'; +import { join } from 'path'; +import ora from 'ora'; +import { PrismaClient } from '@prisma/client'; +import DTF from '@eartharoid/dtf'; + +config(); + +const dtf = new DTF('en-GB'); + +let spinner = ora('Connecting').start(); + +fse.ensureDirSync(join(process.cwd(), './user/dumps')); +const file_path = join(process.cwd(), './user/dumps', `${dtf.fill('YYYY-MM-DD-HH-mm-ss')}-db.json`); + +const prisma_options = {}; + +if (process.env.DB_PROVIDER === 'sqlite' && !process.env.DB_CONNECTION_URL) { + prisma_options.datasources = { db: { url: 'file:' + join(process.cwd(), './user/database.db') } }; +} + +const prisma = new PrismaClient(prisma_options); + +if (process.env.DB_PROVIDER === 'sqlite') { + const { default: sqliteMiddleware } = await import('../src/lib/middleware/prisma-sqlite.js'); + prisma.$use(sqliteMiddleware); + await prisma.$queryRaw`PRAGMA journal_mode=WAL;`; + await prisma.$queryRaw`PRAGMA synchronous=normal;`; +} + +spinner.succeed('Connected'); + +export const models = [ + 'user', + 'guild', + 'tag', + 'category', + 'question', + 'ticket', + 'feedback', + 'questionAnswer', + 'archivedChannel', + 'archivedRole', + 'archivedUser', + 'archivedMessage', +]; + +const dump = await Promise.all( + models.map(async model => { + spinner = ora(`Exporting ${model}`).start(); + const data = await prisma[model].findMany(); + spinner.succeed(`Exported ${data.length} from ${model}`); + return [model, data]; + }), +); + +spinner = ora('Writing').start(); +await fse.promises.writeFile(file_path, JSON.stringify(dump)); +spinner.succeed(`Written to "${file_path}"`); +process.exit(0); diff --git a/scripts/export.mjs b/scripts/export.mjs deleted file mode 100644 index be126b2..0000000 --- a/scripts/export.mjs +++ /dev/null @@ -1,117 +0,0 @@ -import { config } from 'dotenv'; -import { program } from 'commander'; -import fse from 'fs-extra'; -import { join } from 'path'; -import ora from 'ora'; -import { PrismaClient } from '@prisma/client'; -import { createHash } from 'crypto'; -import Cryptr from 'cryptr'; - -config(); - -program - .requiredOption('-g, --guild ', 'the ID of the guild to export'); - -program.parse(); - -const options = program.opts(); - -const hash = createHash('sha256').update(options.guild).digest('hex'); -const file_path = join(process.cwd(), './user/dumps', `${hash}.dump`); -const file_cryptr = new Cryptr(options.guild); -const db_cryptr = new Cryptr(process.env.ENCRYPTION_KEY); - -function decryptIfExists(encrypted) { - if (encrypted) return db_cryptr.decrypt(encrypted); - return null; -} - -fse.ensureDirSync(join(process.cwd(), './user/dumps')); - -let spinner = ora('Connecting').start(); - -const prisma_options = {}; - -if (process.env.DB_PROVIDER === 'sqlite' && !process.env.DB_CONNECTION_URL) { - prisma_options.datasources = { db: { url: 'file:' + join(process.cwd(), './user/database.db') } }; -} - -const prisma = new PrismaClient(prisma_options); - -if (process.env.DB_PROVIDER === 'sqlite') { - const { default: sqliteMiddleware } = await import('../src/lib/middleware/prisma-sqlite.js'); - prisma.$use(sqliteMiddleware); - await prisma.$queryRaw`PRAGMA journal_mode=WAL;`; - await prisma.$queryRaw`PRAGMA synchronous=normal;`; -} - -spinner.succeed('Connected'); - - -const dump = {}; - -spinner = ora('Exporting settings').start(); -dump.settings = await prisma.guild.findFirst({ where: { id: options.guild } }); -spinner.succeed('Exported settings'); - -spinner = ora('Exporting categories').start(); -dump.categories = await prisma.category.findMany({ - include: { questions: true }, - where: { guildId: options.guild }, -}); -spinner.succeed(`Exported ${dump.categories.length} categories`); - -spinner = ora('Exporting tags').start(); -dump.tags = await prisma.tag.findMany({ where: { guildId: options.guild } }); -spinner.succeed(`Exported ${dump.tags.length} tags`); - -spinner = ora('Exporting tickets').start(); -dump.tickets = await prisma.ticket.findMany({ - include: { - archivedChannels: true, - archivedMessages: true, - archivedRoles: true, - archivedUsers: true, - feedback: true, - questionAnswers: true, - }, - where: { guildId: options.guild }, -}); -dump.tickets = dump.tickets.map(ticket => { - if (ticket.topic) ticket.topic = decryptIfExists(ticket.topic); - - ticket.archivedChannels = ticket.archivedChannels.map(channel => { - channel.name = decryptIfExists(channel.name); - return channel; - }); - - ticket.archivedMessages = ticket.archivedMessages.map(message => { - message.content = decryptIfExists(message.content); - return message; - }); - - ticket.archivedUsers = ticket.archivedUsers.map(user => { - user.displayName = decryptIfExists(user.displayName); - user.username = decryptIfExists(user.username); - return user; - }); - - if (ticket.feedback?.comment) { - ticket.feedback.comment = decryptIfExists(ticket.feedback.comment); - } - - ticket.questionAnswers = ticket.questionAnswers.map(answer => { - if (answer.value) answer.value = decryptIfExists(answer.value); - return answer; - }); - - return ticket; -}); -spinner.succeed(`Exported ${dump.tickets.length} tickets`); - -spinner = ora(`Writing to "${file_path}"`).start(); - -// async to not freeze the spinner -await fse.promises.writeFile(file_path, file_cryptr.encrypt(JSON.stringify(dump))); - -spinner.succeed(`Written to "${file_path}"`); diff --git a/scripts/import.mjs b/scripts/import.mjs deleted file mode 100644 index 718507a..0000000 --- a/scripts/import.mjs +++ /dev/null @@ -1,208 +0,0 @@ -import { config } from 'dotenv'; -import { program } from 'commander'; -import fse from 'fs-extra'; -import { join } from 'path'; -import ora from 'ora'; -import { PrismaClient } from '@prisma/client'; -import { createHash } from 'crypto'; -import Cryptr from 'cryptr'; - -config(); - -program - .requiredOption('-g, --guild ', 'the ID of the guild to export') - .option('-f, --force', 'DELETE all data if the guild already exists', false); - -program.parse(); - -const options = program.opts(); - -const hash = createHash('sha256').update(options.guild).digest('hex'); -const file_cryptr = new Cryptr(options.guild); -const db_cryptr = new Cryptr(process.env.ENCRYPTION_KEY); - -function encryptIfExists(plain_text) { - if (plain_text) return db_cryptr.encrypt(plain_text); - return null; -} - -let spinner = ora('Connecting').start(); - -const prisma_options = {}; - -if (process.env.DB_PROVIDER === 'sqlite' && !process.env.DB_CONNECTION_URL) { - prisma_options.datasources = { db: { url: 'file:' + join(process.cwd(), './user/database.db') } }; -} - -const prisma = new PrismaClient(prisma_options); - -if (process.env.DB_PROVIDER === 'sqlite') { - const { default: sqliteMiddleware } = await import('../src/lib/middleware/prisma-sqlite.js'); - prisma.$use(sqliteMiddleware); - await prisma.$queryRaw`PRAGMA journal_mode=WAL;`; - await prisma.$queryRaw`PRAGMA synchronous=normal;`; -} - -spinner.succeed('Connected'); - -spinner = ora('Reading dump file').start(); -const file_path = join(process.cwd(), './user/dumps', `${hash}.dump`); -const dump = JSON.parse(file_cryptr.decrypt(await fse.promises.readFile(file_path, 'utf8'))); -spinner.succeed('Read dump file'); - -spinner = ora('Checking if guild exists').start(); -const exists = await prisma.guild.count({ where: { id: options.guild } }); -if (exists === 0) { - spinner.succeed('Guild doesn\'t exist'); -} else { - if (options.force) { - await prisma.guild.delete({ where: { id: options.guild } }); - spinner.succeed('Deleted guild'); - } else { - spinner.fail('Guild already exists; run again with --force to delete it'); - process.exit(1); - } -} - -spinner = ora('Importing settings & tags').start(); -await prisma.guild.create({ - data: { - ...dump.settings, - id: options.guild, - tags: { - create: dump.tags.map(tag => { - delete tag.guildId; - return tag; - }), - }, - }, -}); -spinner.succeed(`Imported settings & ${dump.tags.length} tags`); - -const category_map = {}; -spinner = ora('Importing categories').start(); -for (const category of dump.categories) { - const original_id = category.id; - delete category.id; - delete category.guildId; - category.questions = { - create: category.questions.map(question => { - delete question.categoryId; - return question; - }), - }; - const { id: new_id } = await prisma.category.create({ - data: { - ...category, - guild: { connect: { id: options.guild } }, - }, - }); - category_map[original_id] = new_id; -} -spinner.succeed(`Imported ${dump.categories.length} categories`); - -spinner = ora('Importing tickets').start(); -for (const i in dump.tickets) { - spinner.text = `Importing tickets (${i}/${dump.tickets.length})`; - const ticket = dump.tickets[i]; - ticket.category = { connect: { id: category_map[ticket.categoryId] } }; - - if (ticket.topic) ticket.topic = encryptIfExists(ticket.topic); - - ticket.archivedChannels = { - create: ticket.archivedChannels.map(channel => { - delete channel.ticketId; - channel.name = encryptIfExists(channel.name); - return channel; - }), - }; - - ticket.archivedUsers = { - create: ticket.archivedUsers.map(user => { - delete user.ticketId; - user.displayName = encryptIfExists(user.displayName); - user.username = encryptIfExists(user.username); - return user; - }), - }; - - ticket.archivedRoles = { - create: ticket.archivedRoles.map(role => { - delete role.ticketId; - return role; - }), - }; - - const archivedMessages = ticket.archivedMessages.map(message => { - message.content = encryptIfExists(message.content); - return message; - }); - ticket.archivedMessages = undefined; - - if (ticket.feedback) { - delete ticket.feedback.ticketId; - delete ticket.feedback.guildId; - ticket.feedback.guild = { connect: { id: options.guild } }; - if (ticket.feedback.comment) { - ticket.feedback.comment = encryptIfExists(ticket.feedback.comment); - } - ticket.feedback = { create: ticket.feedback }; - } else { - ticket.feedback = undefined; - } - - if (ticket.questionAnswers?.length) { - ticket.questionAnswers = { - createMany: ticket.questionAnswers.map(answer => { - delete answer.ticketId; - if (answer.value) answer.value = encryptIfExists(answer.value); - return answer; - }), - }; - } else { - ticket.questionAnswers = undefined; - } - - if (ticket.claimedById) { - ticket.claimedBy = { - connectOrCreate: { - create: { id: ticket.claimedById }, - where: { id: ticket.claimedById }, - }, - }; - } - if (ticket.closedById) { - ticket.closedBy = { - connectOrCreate: { - create: { id: ticket.closedById }, - where: { id: ticket.closedById }, - }, - }; - } - if (ticket.createdById) { - ticket.createdBy = { - connectOrCreate: { - create: { id: ticket.createdById }, - where: { id: ticket.createdById }, - }, - }; - } - - - - if (ticket.referencesTicketId) { - ticket.referencesTicket = { connect: { id: ticket.referencedTicketId } }; - } - ticket.guild = { connect: { id: options.guild } }; - - delete ticket.categoryId; - delete ticket.guildId; - delete ticket.claimedById; - delete ticket.closedById; - delete ticket.createdById; - delete ticket.referencesTicketId; - - await prisma.ticket.create({ data: ticket }); - await prisma.archivedMessage.createMany({ data: archivedMessages }); -} -spinner.succeed(`Imported ${dump.tickets.length} tickets`); diff --git a/scripts/restore.mjs b/scripts/restore.mjs new file mode 100644 index 0000000..c628cd7 --- /dev/null +++ b/scripts/restore.mjs @@ -0,0 +1,51 @@ +import { config } from 'dotenv'; +import { program } from 'commander'; +import fse from 'fs-extra'; +import { join } from 'path'; +import ora from 'ora'; +import { PrismaClient } from '@prisma/client'; + +config(); + +program + .requiredOption('-f, --file ', 'the path of the dump to import') + .requiredOption('-y, --yes', 'yes, DELETE EVERYTHING in the database'); + +program.parse(); + +const options = program.opts(); + +let spinner = ora('Connecting').start(); + +const prisma_options = {}; + +if (process.env.DB_PROVIDER === 'sqlite' && !process.env.DB_CONNECTION_URL) { + prisma_options.datasources = { db: { url: 'file:' + join(process.cwd(), './user/database.db') } }; +} + +const prisma = new PrismaClient(prisma_options); + +if (process.env.DB_PROVIDER === 'sqlite') { + const { default: sqliteMiddleware } = await import('../src/lib/middleware/prisma-sqlite.js'); + prisma.$use(sqliteMiddleware); + await prisma.$queryRaw`PRAGMA journal_mode=WAL;`; + await prisma.$queryRaw`PRAGMA synchronous=normal;`; +} + +spinner.succeed('Connected'); + +spinner = ora(`Reading ${options.file}`).start(); +const dump = JSON.parse(await fse.promises.readFile(options.file, 'utf8')); +spinner.succeed(`Parsed ${options.file}`); + +// ! this order is important +const queries = [ + prisma.guild.deleteMany(), + prisma.user.deleteMany(), +]; + +for (const [model, data] of dump) queries.push(prisma[model].createMany({ data })); +spinner = ora('Importing').start(); +const [,, ...results] = await prisma.$transaction(queries); +for (const idx in results) spinner.succeed(`Imported ${results[idx].count} into ${dump[idx][0]}`); +process.exit(0);