Talon
/
notebrook-notes
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Switch backend to node, docker likely broken

main
Talon 2024-08-26 00:18:00 +02:00
parent bf1719d75d
commit b487ab82cc
15 changed files with 7101 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
backend/bun.lockb
View File

Binary file not shown.

BIN
backend/db.sqlite 100644
View File

Binary file not shown.

9
backend/src/migrations/0_init.sql → backend/migrations/1_init.sql
View File

@ -2,7 +2,8 @@ CREATE TABLE IF NOT EXISTS channels (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL,
createdAt DATETIME DEFAULT CURRENT_TIMESTAMP
) CREATE TABLE IF NOT EXISTS files (
);
CREATE TABLE IF NOT EXISTS files (
id INTEGER PRIMARY KEY AUTOINCREMENT,
channelId INTEGER,
filePath TEXT,
@ -11,7 +12,8 @@ CREATE TABLE IF NOT EXISTS channels (
originalName TEXT,
createdAt DATETIME DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (channelId) REFERENCES channels (id) ON DELETE CASCADE
) CREATE TABLE IF NOT EXISTS messages (
);
CREATE TABLE IF NOT EXISTS messages (
id INTEGER PRIMARY KEY AUTOINCREMENT,
channelId INTEGER,
content TEXT,
@ -21,7 +23,8 @@ CREATE TABLE IF NOT EXISTS channels (
FOREIGN KEY (fileId) REFERENCES files (id) ON DELETE
SET
NULL
) CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5(
);
CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5(
content,
content = 'messages',
content_rowid = 'id'

0
backend/src/migrations/1_localtime.sql → backend/migrations/2_localtime.sql
View File

2084
backend/package-lock.json generated 100644
View File

File diff suppressed because it is too large Load Diff

9
backend/package.json
View File

@ -3,8 +3,8 @@
"module": "src/server.ts",
"type": "module",
"scripts": {
"start": "bun run src/server.ts",
"dev": "bun run --watch src/server.ts",
"start": "tsx src/server.ts",
"dev": "tsx --watch src/server.ts",
"test": "echo \"Error: no test specified\" && exit 1"
},
"devDependencies": {
@ -20,12 +20,15 @@
"@types/jsonwebtoken": "^9.0.6",
"@types/multer": "^1.4.11",
"@types/ws": "^8.5.12",
"better-sqlite3": "^11.2.1",
"cors": "^2.8.5",
"dotenv": "^16.4.5",
"express": "^4.19.2",
"multer": "^1.4.5-lts.1",
"ollama": "^0.5.8",
"openai": "^4.56.0",
"sharp": "^0.33.5",
"tsx": "^4.18.0",
"ws": "^8.18.0"
}
}
}

3
backend/src/config.ts
View File

@ -1,3 +1,6 @@
import dotenv from "dotenv";
dotenv.config();
export const DB_PATH = process.env["DB_PATH"] || "/usr/src/app/data/db.sqlite";
export const SECRET_KEY = process.env["API_TOKEN"] || "";
export const UPLOAD_DIR = process.env["UPLOAD_DIR"] || "/usr/src/app/data/uploads/";

29
backend/src/db.ts
View File

@ -1,14 +1,14 @@
import { Database } from 'bun:sqlite';
import Database from 'better-sqlite3';
import { DB_PATH } from './config';
import { logger } from './globals';
import { readdir, readFile } from "fs/promises";
import { join } from "path";
import { join, dirname } from "path";
export let FTS5Enabled = true;
export const initializeDB = () => {
logger.info("Checking fts");
const ftstest = db.query(`pragma compile_options;`);
const ftstest = db.prepare(`pragma compile_options;`);
const result = ftstest.all() as { compile_options: string }[];
if (result.find((o) => o["compile_options"].includes("ENABLE_FTS5"))) {
logger.info("FTS5 is enabled");
@ -27,28 +27,28 @@ export const initializeDB = () => {
export const migrate = async () => {
logger.info(`Checking for migrations...`);
const result = db.query(`SELECT name FROM sqlite_master WHERE type='table' AND name='meta'`);
const result = db.prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name='meta'`);
if (result.all().length === 0) {
logger.info(`Creating meta table...`);
db.run(`CREATE TABLE meta (version INTEGER)`);
db.run(`INSERT INTO meta (version) VALUES (0)`);
db.exec(`CREATE TABLE meta (version INTEGER)`);
db.exec(`INSERT INTO meta (version) VALUES (-1)`);
}
const version = db.query(`SELECT version FROM meta`).get() as { version: number };
const version = db.prepare(`SELECT version FROM meta`).get() as { version: number };
logger.info(`Migration version: ${version.version}`);
// we are in bun.js. use its API's to read the file list.
logger.info(`Searching for migrations in ${join(__dirname, "migrations")}`);
const files = await readdir(join(__dirname, "migrations"));
logger.info(`Searching for migrations in ${join("migrations")}`);
const files = await readdir(join("migrations"));
for (const file of files) {
const [fileVersion, ...rest] = file.split("_");
logger.info(`Found migration ${fileVersion}`);
if (fileVersion && Number(fileVersion) > version.version) {
logger.info(`Running migration ${file}`);
const sql = new TextDecoder().decode(await readFile(join(__dirname, `migrations/${file}`)));
db.run(sql);
const query = db.query(`UPDATE meta SET version = $version`);
const res = query.run({ $version: Number(fileVersion) })
const sql = new TextDecoder().decode(await readFile(join(`migrations/${file}`)));
db.exec(sql);
const query = db.prepare(`UPDATE meta SET version = ($version)`);
const res = query.run({ version: fileVersion })
logger.info(`Migration ${file} done`);
}
}
@ -60,5 +60,8 @@ logger.info(`Loading database at ${DB_PATH}`);
export const db = new Database(DB_PATH);
initializeDB();
migrate();

2
backend/src/jobs/vacuum.ts
View File

@ -4,6 +4,6 @@ import { db } from "../db";
export const scheduleVacuum = () => {
scheduler.register('vacuum', () => {
db.query('VACUUM');
db.exec('VACUUM');
}, 1, TimeUnit.DAY);
}

18
backend/src/services/channel-service.ts
View File

@ -2,36 +2,36 @@ import { db } from "../db";
import { events } from "../globals";
export const createChannel = async (name: string) => {
const query = db.query(`INSERT INTO channels (name) VALUES ($name)`);
const result = query.run({ $name: name });
const query = db.prepare(`INSERT INTO channels (name) VALUES ($name)`);
const result = query.run({ name: name });
events.emit('channel-created', { id: result.lastInsertRowid, name });
return { id: result.lastInsertRowid, name };
}
export const deleteChannel = async (id: string) => {
const query = db.query(`DELETE FROM channels WHERE id = ($channelId)`);
const result = query.run({$channelId: id});
const query = db.prepare(`DELETE FROM channels WHERE id = ($channelId)`);
const result = query.run({channelId: id});
// No need to manually delete messages and files as they are set to cascade on delete in the schema
events .emit('channel-deleted', id);
return result;
}
export const getChannels = async () => {
const query = db.query(`SELECT * FROM channels`);
const query = db.prepare(`SELECT * FROM channels`);
const rows = query.all();
return rows;
}
export const mergeChannel = async (channelId: string, targetChannelId: string) => {
const query = db.query(`UPDATE messages SET channelId = $targetChannelId WHERE channelId = $channelId`);
const result = query.run({ $channelId: channelId, $targetChannelId: targetChannelId });
const query = db.prepare(`UPDATE messages SET channelId = $targetChannelId WHERE channelId = $channelId`);
const result = query.run({ channelId: channelId, targetChannelId: targetChannelId });
events.emit('channel-merged', channelId, targetChannelId);
return result;
}
export const updateChannel = async (id: string, name: string) => {
const query = db.query(`UPDATE channels SET name = $name WHERE id = $id`);
const result = query.run({ $id: id, $name: name });
const query = db.prepare(`UPDATE channels SET name = $name WHERE id = $id`);
const result = query.run({ id: id, name: name });
events.emit('channel-updated', id, name);
return result;
}

12
backend/src/services/file-service.ts
View File

@ -2,20 +2,20 @@ import { db } from "../db";
import { events } from "../globals";
export const uploadFile = async (channelId: string, messageId: string, filePath: string, fileType: string, fileSize: number, originalName: string) => {
const query = db.query(`INSERT INTO files (channelId, filePath, fileType, fileSize, originalName) VALUES ($channelId, $filePath, $fileType, $fileSize, $originalName)`);
const result = query.run({ $channelId: channelId, $filePath: filePath, $fileType: fileType, $fileSize: fileSize, $originalName: originalName });
const query = db.prepare(`INSERT INTO files (channelId, filePath, fileType, fileSize, originalName) VALUES ($channelId, $filePath, $fileType, $fileSize, $originalName)`);
const result = query.run({ channelId: channelId, filePath: filePath, fileType: fileType, fileSize: fileSize, originalName: originalName });
const fileId = result.lastInsertRowid;
const updateQuery = db.query(`UPDATE messages SET fileId = $fileId WHERE id = $messageId`);
const result2 = updateQuery.run({ $fileId: fileId, $messageId: messageId });
const updateQuery = db.prepare(`UPDATE messages SET fileId = $fileId WHERE id = $messageId`);
const result2 = updateQuery.run({ fileId: fileId, messageId: messageId });
events.emit('file-uploaded', result.lastInsertRowid, channelId, messageId, filePath, fileType, fileSize, originalName);
return result2; ''
}
export const getFiles = async (messageId: string) => {
const query = db.query(`SELECT * FROM files WHERE messageId = $messageId`);
const rows = query.all({ $messageId: messageId });
const query = db.prepare(`SELECT * FROM files WHERE messageId = $messageId`);
const rows = query.all({ messageId: messageId });
return rows;
}

32
backend/src/services/message-service.ts
View File

@ -2,15 +2,15 @@ import { db, FTS5Enabled } from "../db";
import { events } from "../globals";
export const createMessage = async (channelId: string, content: string) => {
const query = db.query(`INSERT INTO messages (channelId, content) VALUES ($channelId, $content)`);
const result = query.run({ $channelId: channelId, $content: content });
const query = db.prepare(`INSERT INTO messages (channelId, content) VALUES ($channelId, $content)`);
const result = query.run({ channelId: channelId, content: content });
const messageId = result.lastInsertRowid;
console.log(`Adding message for search with id ${messageId}`);
// Insert into FTS table if FTS is enabled.
if (FTS5Enabled) {
const query2 = db.query(`INSERT INTO messages_fts (rowid, content) VALUES ($rowId, $content)`);
const result2 = query2.run({ $rowId: messageId, $content: content });
const query2 = db.prepare(`INSERT INTO messages_fts (rowid, content) VALUES ($rowId, $content)`);
const result2 = query2.run({ rowId: messageId, content: content });
}
events.emit('message-created', messageId, channelId, content);
@ -18,36 +18,36 @@ export const createMessage = async (channelId: string, content: string) => {
}
export const updateMessage = async (messageId: string, content: string, append: boolean = false) => {
const query = db.query(`UPDATE messages SET content = $content WHERE id = $id`);
const result = query.run({ $content: content, $id: messageId });
const query = db.prepare(`UPDATE messages SET content = $content WHERE id = $id`);
const result = query.run({ content: content, id: messageId });
// Update FTS table if enabled
if (!FTS5Enabled) {
const query2 = db.query(`INSERT INTO messages_fts (rowid, content) VALUES ($rowId, $content) ON CONFLICT(rowid) DO UPDATE SET content = excluded.content`);
const result2 = query.run({ $rowId: messageId, $content: content });
const query2 = db.prepare(`INSERT INTO messages_fts (rowid, content) VALUES ($rowId, $content) ON CONFLICT(rowid) DO UPDATE SET content = excluded.content`);
const result2 = query.run({ rowId: messageId, content: content });
}
events.emit('message-updated', messageId, content);
return result;
}
export const deleteMessage = async (messageId: string) => {
const query = db.query(`DELETE FROM messages WHERE id = $id`);
const result = query.run({ $id: messageId });
const query = db.prepare(`DELETE FROM messages WHERE id = $id`);
const result = query.run({ id: messageId });
// Remove from FTS table if enabled
if (FTS5Enabled) {
const query2 = db.query(`DELETE FROM messages_fts WHERE rowid = $rowId`);
const result2 = query.run({ $rowId: messageId });
const query2 = db.prepare(`DELETE FROM messages_fts WHERE rowid = $rowId`);
const result2 = query.run({ rowId: messageId });
}
events.emit('message-deleted', messageId);
return result;
}
export const getMessages = async (channelId: string) => {
const query = db.query(`
const query = db.prepare(`
SELECT
messages.id, messages.channelId, messages.content, messages.createdAt,
files.id as fileId, files.filePath, files.fileType, files.createdAt as fileCreatedAt, files.originalName, files.fileSize
@ -60,12 +60,12 @@ export const getMessages = async (channelId: string) => {
WHERE
messages.channelId = $channelId
`);
const rows = query.all({ $channelId: channelId });
const rows = query.all({ channelId: channelId });
return rows;
}
export const getMessage = async (id: string) => {
const query = db.query(`
const query = db.prepare(`
SELECT
messages.id, messages.channelId, messages.content, messages.createdAt,
files.id as fileId, files.filePath, files.fileType, files.createdAt as fileCreatedAt, files.originalName, files.fileSize
@ -78,6 +78,6 @@ export const getMessage = async (id: string) => {
WHERE
messages.id = $id
`);
const row = query.get({ $id: id });
const row = query.get({ id: id });
return row;
}

10
backend/src/services/search-service.ts
View File

@ -12,7 +12,7 @@ export const search = async (query: string, channelId?: string) => {
JOIN messages ON messages_fts.rowid = messages.id
WHERE messages_fts MATCH lower($query) AND messages.channelId = $channelId
`;
params = { $channelId: channelId, $query: (query || '').toString().toLowerCase() };
params = { channelId: channelId, query: (query || '').toString().toLowerCase() };
} else {
sql = `
SELECT messages.id, messages.channelId, messages.content, messages.createdAt
@ -20,7 +20,7 @@ export const search = async (query: string, channelId?: string) => {
JOIN messages ON messages_fts.rowid = messages.id
WHERE messages_fts MATCH lower($query)
`;
params = { $query: (query || '').toString().toLowerCase() };
params = { query: (query || '').toString().toLowerCase() };
}
} else {
console.log("Performing search without FTS5. This might be very slow.");
@ -28,16 +28,16 @@ export const search = async (query: string, channelId?: string) => {
sql = `
SELECT * FROM messages WHERE LOWER(content) LIKE '%' || LOWER($query) || '%' AND channelId = $channelId
`;
params = { $channelId: channelId, $query: query };
params = { channelId: channelId, query: query };
} else {
sql = `
SELECT * FROM messages WHERE LOWER(content) LIKE '%' || LOWER($query) || '%'
`;
params = { $query: query };
params = { query: query };
}
}
const sqlquery = db.query(sql);
const sqlquery = db.prepare(sql);
const rows = sqlquery.all(params);
return rows;

BIN
frontend/bun.lockb
View File

Binary file not shown.

4949
frontend/package-lock.json generated 100644
View File

File diff suppressed because it is too large Load Diff