Add mantis summaries and start of email summaries... Need to figure out a way to get the emails since MS block IMAP :(

This commit is contained in:
Cameron Redmore 2025-04-23 21:12:08 +01:00
parent 2d11d0bd79
commit 2ad9a63582
18 changed files with 1993 additions and 577 deletions

View file

@ -1,110 +1,14 @@
import Database from 'better-sqlite3';
import { join } from 'path';
import { fileURLToPath } from 'url';
import fs from 'fs'; // Needed to check if db file exists
import { PrismaClient } from '@prisma/client';
// Determine the database path relative to this file
const __dirname = fileURLToPath(new URL('.', import.meta.url));
const dbPath = join(__dirname, 'forms.db');
// Instantiate Prisma Client
const prisma = new PrismaClient();
let db = null;
// Export the Prisma Client instance for use in other modules
export default prisma;
export function initializeDatabase() {
if (db) {
return db;
}
// --- Old better-sqlite3 code removed ---
// No need for initializeDatabase, getDb, closeDatabase, etc.
// Prisma Client manages the connection pool.
try {
// Check if the directory exists, create if not (better-sqlite3 might need this)
const dbDir = join(__dirname);
if (!fs.existsSync(dbDir)) {
fs.mkdirSync(dbDir, { recursive: true });
}
// better-sqlite3 constructor opens/creates the database file
db = new Database(dbPath, { verbose: console.log }); // Enable verbose logging
console.log('Connected to the SQLite database using better-sqlite3.');
// Ensure WAL mode is enabled for better concurrency
db.pragma('journal_mode = WAL');
// Create tables if they don't exist (run sequentially)
db.exec(`
CREATE TABLE IF NOT EXISTS forms (
id INTEGER PRIMARY KEY AUTOINCREMENT,
title TEXT NOT NULL,
description TEXT,
createdAt DATETIME DEFAULT CURRENT_TIMESTAMP
);
`);
db.exec(`
CREATE TABLE IF NOT EXISTS categories (
id INTEGER PRIMARY KEY AUTOINCREMENT,
formId INTEGER NOT NULL,
name TEXT NOT NULL,
sortOrder INTEGER DEFAULT 0,
FOREIGN KEY (formId) REFERENCES forms (id) ON DELETE CASCADE
);
`);
db.exec(`
CREATE TABLE IF NOT EXISTS fields (
id INTEGER PRIMARY KEY AUTOINCREMENT,
categoryId INTEGER NOT NULL,
label TEXT NOT NULL,
type TEXT NOT NULL CHECK(type IN ('text', 'number', 'date', 'textarea', 'boolean')),
description TEXT,
sortOrder INTEGER NOT NULL,
FOREIGN KEY (categoryId) REFERENCES categories(id) ON DELETE CASCADE
);
`);
db.exec(`
CREATE TABLE IF NOT EXISTS responses (
id INTEGER PRIMARY KEY AUTOINCREMENT,
formId INTEGER NOT NULL,
submittedAt DATETIME DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (formId) REFERENCES forms (id) ON DELETE CASCADE
);
`);
db.exec(`
CREATE TABLE IF NOT EXISTS response_values (
id INTEGER PRIMARY KEY AUTOINCREMENT,
responseId INTEGER NOT NULL,
fieldId INTEGER NOT NULL,
value TEXT,
FOREIGN KEY (responseId) REFERENCES responses (id) ON DELETE CASCADE,
FOREIGN KEY (fieldId) REFERENCES fields (id) ON DELETE CASCADE
);
`);
console.log('Database tables ensured.');
return db;
} catch (err) {
console.error('Error initializing database with better-sqlite3:', err.message);
throw err; // Re-throw the error
}
}
export function getDb() {
if (!db) {
// Try to initialize if not already done (e.g., during hot reload)
try {
initializeDatabase();
} catch (err) {
throw new Error('Database not initialized and initialization failed.');
}
if (!db) { // Check again after trying to initialize
throw new Error('Database not initialized. Call initializeDatabase first.');
}
}
return db;
}
// Optional: Add a function to close the database gracefully on server shutdown
export function closeDatabase() {
if (db) {
db.close();
db = null;
console.log('Database connection closed.');
}
}
// --- Settings Functions removed ---
// Settings can now be accessed via prisma.setting.findUnique, prisma.setting.upsert, etc.

File diff suppressed because it is too large Load diff

View file

@ -19,9 +19,10 @@ import {
defineSsrRenderPreloadTag
} from '#q-app/wrappers'
// Import database initialization and close function
import { initializeDatabase, closeDatabase } from './database.js';
import prisma from './database.js'; // Import the prisma client instance
import apiRoutes from './routes/api.js';
import cron from 'node-cron';
import { generateAndStoreMantisSummary } from './services/mantisSummarizer.js';
/**
* Create your webserver and return its instance.
@ -35,12 +36,31 @@ export const create = defineSsrCreate((/* { ... } */) => {
// Initialize the database (now synchronous)
try {
initializeDatabase();
console.log('Database initialized successfully.');
console.log('Prisma Client is ready.'); // Log Prisma readiness
// Schedule the Mantis summary task after DB initialization
// Run daily at 1:00 AM server time (adjust as needed)
cron.schedule('0 1 * * *', async () => {
console.log('Running scheduled Mantis summary task...');
try {
await generateAndStoreMantisSummary();
console.log('Scheduled Mantis summary task completed.');
} catch (error) {
console.error('Error running scheduled Mantis summary task:', error);
}
}, {
scheduled: true,
timezone: "Europe/London" // Example: Set to your server's timezone
});
console.log('Mantis summary cron job scheduled.');
// Optional: Run once immediately on server start if needed
generateAndStoreMantisSummary().catch(err => console.error('Initial Mantis summary failed:', err));
} catch (error) {
console.error('Failed to initialize database:', error);
console.error('Error during server setup:', error);
// Optionally handle the error more gracefully, e.g., prevent server start
process.exit(1); // Exit if DB connection fails
process.exit(1); // Exit if setup fails
}
// attackers can use this header to detect apps running Express
@ -91,9 +111,14 @@ export const listen = defineSsrListen(({ app, devHttpsApp, port }) => {
*
* Can be async: defineSsrClose(async ({ ... }) => { ... })
*/
export const close = defineSsrClose(({ listenResult }) => {
export const close = defineSsrClose(async ({ listenResult }) => {
// Close the database connection when the server shuts down
closeDatabase();
try {
await prisma.$disconnect();
console.log('Prisma Client disconnected.');
} catch (e) {
console.error('Error disconnecting Prisma Client:', e);
}
return listenResult.close()
})

View file

@ -0,0 +1,199 @@
import Imap from 'node-imap';
import { simpleParser } from 'mailparser';
import { GoogleGenAI } from '@google/genai';
import prisma from '../database.js';
// --- Environment Variables ---
const { GOOGLE_API_KEY } = process.env; // Added
// --- AI Setup ---
const ai = GOOGLE_API_KEY ? new GoogleGenAI({
apiKey: GOOGLE_API_KEY,
}) : null; // Added
export async function fetchAndFormatEmails() {
return new Promise((resolve, reject) => {
const imapConfig = {
user: process.env.OUTLOOK_EMAIL_ADDRESS,
password: process.env.OUTLOOK_APP_PASSWORD,
host: 'outlook.office365.com',
port: 993,
tls: true,
tlsOptions: { rejectUnauthorized: false } // Adjust as needed for your environment
};
const imap = new Imap(imapConfig);
const emailsJson = [];
function openInbox(cb) {
// Note: IMAP uses '/' as hierarchy separator, adjust if your server uses something else
imap.openBox('SLSNotifications/Reports/Backups', false, cb);
}
imap.once('ready', () => {
openInbox((err, box) => {
if (err) {
imap.end();
return reject(new Error(`Error opening mailbox: ${err.message}`));
}
const yesterday = new Date();
yesterday.setDate(yesterday.getDate() - 1);
const searchCriteria = [['SINCE', yesterday.toISOString().split('T')[0]]]; // Search since midnight yesterday
const fetchOptions = { bodies: ['HEADER.FIELDS (SUBJECT DATE)', 'TEXT'], struct: true };
imap.search(searchCriteria, (searchErr, results) => {
if (searchErr) {
imap.end();
return reject(new Error(`Error searching emails: ${searchErr.message}`));
}
if (results.length === 0) {
console.log('No emails found from the last 24 hours.');
imap.end();
return resolve([]);
}
const f = imap.fetch(results, fetchOptions);
let processedCount = 0;
f.on('message', (msg, seqno) => {
let header = '';
let body = '';
msg.on('body', (stream, info) => {
let buffer = '';
stream.on('data', (chunk) => {
buffer += chunk.toString('utf8');
});
stream.once('end', () => {
if (info.which === 'TEXT') {
body = buffer;
} else {
// Assuming HEADER.FIELDS (SUBJECT DATE) comes as one chunk
header = buffer;
}
});
});
msg.once('attributes', (attrs) => {
// Attributes might contain date if not fetched via header
});
msg.once('end', async () => {
try {
// Use mailparser to handle potential encoding issues and structure
const mail = await simpleParser(`Subject: ${header.match(/Subject: (.*)/i)?.[1] || ''}\nDate: ${header.match(/Date: (.*)/i)?.[1] || ''}\n\n${body}`);
emailsJson.push({
title: mail.subject || 'No Subject',
time: mail.date ? mail.date.toISOString() : 'No Date',
body: mail.text || mail.html || 'No Body Content' // Prefer text, fallback to html, then empty
});
} catch (parseErr) {
console.error(`Error parsing email seqno ${seqno}:`, parseErr);
// Decide if you want to reject or just skip this email
}
processedCount++;
if (processedCount === results.length) {
// This check might be slightly inaccurate if errors occur,
// but it's a common pattern. Consider refining with promises.
}
});
});
f.once('error', (fetchErr) => {
console.error('Fetch error: ' + fetchErr);
// Don't reject here immediately, might still get some emails
});
f.once('end', () => {
console.log('Done fetching all messages!');
imap.end();
});
});
});
});
imap.once('error', (err) => {
reject(new Error(`IMAP Connection Error: ${err.message}`));
});
imap.once('end', () => {
console.log('IMAP Connection ended.');
resolve(emailsJson); // Resolve with the collected emails
});
imap.connect();
});
}
// --- Email Summary Logic (New Function) ---
export async function generateAndStoreEmailSummary() {
console.log('Attempting to generate and store Email summary...');
if (!ai) {
console.error('Google AI API key not configured. Skipping email summary generation.');
return;
}
try {
// Get the prompt from the database settings using Prisma
const setting = await prisma.setting.findUnique({
where: { key: 'emailPrompt' }, // Use 'emailPrompt' as the key
select: { value: true }
});
const promptTemplate = setting?.value;
if (!promptTemplate) {
console.error('Email prompt not found in database settings (key: emailPrompt). Skipping summary generation.');
return;
}
const emails = await fetchAndFormatEmails();
let summaryText;
if (emails.length === 0) {
summaryText = "No relevant emails found in the last 24 hours.";
console.log('No recent emails found for summary.');
} else {
console.log(`Found ${emails.length} recent emails. Generating summary...`);
// Replace placeholder in the prompt template
// Ensure your prompt template uses $EMAIL_DATA
let prompt = promptTemplate.replaceAll("$EMAIL_DATA", JSON.stringify(emails, null, 2));
// Call the AI model (adjust model name and config as needed)
const response = await ai.models.generateContent({
"model": "gemini-2.5-preview-04-17",
"contents": prompt,
config: {
temperature: 0 // Adjust temperature as needed
}
});
summaryText = response.text;
console.log('Email summary generated successfully by AI.');
}
// Store the summary in the database using Prisma upsert
const today = new Date();
today.setUTCHours(0, 0, 0, 0); // Use UTC start of day for consistency
await prisma.emailSummary.upsert({
where: { summaryDate: today },
update: {
summaryText: summaryText,
// generatedAt is updated automatically by @default(now())
},
create: {
summaryDate: today,
summaryText: summaryText,
},
});
console.log(`Email summary for ${today.toISOString().split('T')[0]} stored/updated in the database.`);
} catch (error) {
console.error("Error during Email summary generation/storage:", error);
// Re-throw or handle as appropriate for your application
throw error;
}
}

View file

@ -0,0 +1,171 @@
import axios from 'axios';
import { GoogleGenAI } from '@google/genai';
import prisma from '../database.js'; // Import Prisma client
// --- Environment Variables ---
const {
MANTIS_API_KEY,
MANTIS_API_ENDPOINT,
GOOGLE_API_KEY
} = process.env;
// --- Mantis Summarizer Setup ---
const ai = GOOGLE_API_KEY ? new GoogleGenAI({
apiKey: GOOGLE_API_KEY,
}) : null;
const usernameMap = {
'credmore': 'Cameron Redmore',
'dgibson': 'Dane Gibson',
'egzibovskis': 'Ed Gzibovskis',
'ascotney': 'Amanda Scotney',
'gclough': 'Garry Clough',
'slee': 'Sarah Lee',
'dwalker': 'Dave Walker',
'askaith': 'Amy Skaith',
'dpotter': 'Danny Potter',
'msmart': 'Michael Smart',
// Add other usernames as needed
};
async function getMantisTickets() {
if (!MANTIS_API_ENDPOINT || !MANTIS_API_KEY) {
throw new Error("Mantis API endpoint or key not configured in environment variables.");
}
const url = `${MANTIS_API_ENDPOINT}/issues?project_id=1&page_size=50&select=id,summary,description,created_at,updated_at,reporter,notes`;
const headers = {
'Authorization': `${MANTIS_API_KEY}`,
'Accept': 'application/json',
'Content-Type': 'application/json',
};
try {
const response = await axios.get(url, { headers });
const tickets = response.data.issues.filter((ticket) => {
const ticketDate = new Date(ticket.updated_at);
const thresholdDate = new Date();
const currentDay = thresholdDate.getDay(); // Sunday = 0, Monday = 1, ...
// Go back 4 days if Monday (to include Fri, Sat, Sun), otherwise 2 days
const daysToSubtract = currentDay === 1 ? 4 : 2;
thresholdDate.setDate(thresholdDate.getDate() - daysToSubtract);
thresholdDate.setHours(0, 0, 0, 0); // Start of the day
return ticketDate >= thresholdDate;
}).map((ticket) => {
return {
id: ticket.id,
summary: ticket.summary,
description: ticket.description,
created_at: ticket.created_at,
updated_at: ticket.updated_at,
reporter: usernameMap[ticket.reporter?.username] || ticket.reporter?.name || 'Unknown Reporter', // Safer access
notes: (ticket.notes ? ticket.notes.filter((note) => {
const noteDate = new Date(note.created_at);
const thresholdDate = new Date();
const currentDay = thresholdDate.getDay();
const daysToSubtract = currentDay === 1 ? 4 : 2;
thresholdDate.setDate(thresholdDate.getDate() - daysToSubtract);
thresholdDate.setHours(0, 0, 0, 0); // Start of the day
return noteDate >= thresholdDate;
}) : []).map((note) => {
const reporter = usernameMap[note.reporter?.username] || note.reporter?.name || 'Unknown Reporter'; // Safer access
return {
reporter,
created_at: note.created_at,
text: note.text,
};
}),
};
});
return tickets;
} catch (error) {
console.error("Error fetching Mantis tickets:", error.message);
// Check if it's an Axios error and provide more details
if (axios.isAxiosError(error)) {
console.error("Axios error details:", error.response?.status, error.response?.data);
throw new Error(`Failed to fetch Mantis tickets: ${error.response?.statusText || error.message}`);
}
throw new Error(`Failed to fetch Mantis tickets: ${error.message}`);
}
}
// --- Mantis Summary Logic (Exported) --- //
export async function generateAndStoreMantisSummary() {
console.log('Attempting to generate and store Mantis summary...');
if (!ai) {
console.error('Google AI API key not configured. Skipping summary generation.');
return;
}
try {
// Get the prompt from the database settings using Prisma
const setting = await prisma.setting.findUnique({
where: { key: 'mantisPrompt' },
select: { value: true }
});
const promptTemplate = setting?.value;
if (!promptTemplate) {
console.error('Mantis prompt not found in database settings (key: mantisPrompt). Skipping summary generation.');
return;
}
const tickets = await getMantisTickets();
let summaryText;
if (tickets.length === 0) {
summaryText = "No Mantis tickets updated recently.";
console.log('No recent Mantis tickets found.');
} else {
console.log(`Found ${tickets.length} recent Mantis tickets. Generating summary...`);
let prompt = promptTemplate.replaceAll("$DATE", new Date().toISOString().split('T')[0]);
prompt = prompt.replaceAll("$MANTIS_TICKETS", JSON.stringify(tickets, null, 2));
const response = await ai.models.generateContent({
"model": "gemini-2.5-flash-preview-04-17",
"contents": prompt,
config: {
temperature: 0
}
});
summaryText = response.text;
console.log('Mantis summary generated successfully by AI.');
}
// Store the summary in the database using Prisma upsert
const today = new Date();
today.setUTCHours(0, 0, 0, 0); // Use UTC start of day for consistency
await prisma.mantisSummary.upsert({
where: { summaryDate: today },
update: {
summaryText: summaryText,
// generatedAt is updated automatically by @default(now())
},
create: {
summaryDate: today,
summaryText: summaryText,
},
});
console.log(`Mantis summary for ${today.toISOString().split('T')[0]} stored/updated in the database.`);
} catch (error) {
console.error("Error during Mantis summary generation/storage:", error);
}
}
export async function generateTodaysSummary() {
console.log('Triggering Mantis summary generation via generateTodaysSummary...');
try {
await generateAndStoreMantisSummary();
return { success: true, message: 'Summary generation process initiated.' };
} catch (error) {
console.error('Error occurred within generateTodaysSummary while calling generateAndStoreMantisSummary:', error);
throw new Error('Failed to initiate Mantis summary generation.');
}
}