Add mantis summaries and start of email summaries... Need to figure out a way to get the emails since MS block IMAP :(
This commit is contained in:
parent
2d11d0bd79
commit
2ad9a63582
18 changed files with 1993 additions and 577 deletions
|
@ -1,110 +1,14 @@
|
|||
import Database from 'better-sqlite3';
|
||||
import { join } from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
import fs from 'fs'; // Needed to check if db file exists
|
||||
import { PrismaClient } from '@prisma/client';
|
||||
|
||||
// Determine the database path relative to this file
|
||||
const __dirname = fileURLToPath(new URL('.', import.meta.url));
|
||||
const dbPath = join(__dirname, 'forms.db');
|
||||
// Instantiate Prisma Client
|
||||
const prisma = new PrismaClient();
|
||||
|
||||
let db = null;
|
||||
// Export the Prisma Client instance for use in other modules
|
||||
export default prisma;
|
||||
|
||||
export function initializeDatabase() {
|
||||
if (db) {
|
||||
return db;
|
||||
}
|
||||
// --- Old better-sqlite3 code removed ---
|
||||
// No need for initializeDatabase, getDb, closeDatabase, etc.
|
||||
// Prisma Client manages the connection pool.
|
||||
|
||||
try {
|
||||
// Check if the directory exists, create if not (better-sqlite3 might need this)
|
||||
const dbDir = join(__dirname);
|
||||
if (!fs.existsSync(dbDir)) {
|
||||
fs.mkdirSync(dbDir, { recursive: true });
|
||||
}
|
||||
|
||||
// better-sqlite3 constructor opens/creates the database file
|
||||
db = new Database(dbPath, { verbose: console.log }); // Enable verbose logging
|
||||
|
||||
console.log('Connected to the SQLite database using better-sqlite3.');
|
||||
|
||||
// Ensure WAL mode is enabled for better concurrency
|
||||
db.pragma('journal_mode = WAL');
|
||||
|
||||
// Create tables if they don't exist (run sequentially)
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS forms (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
title TEXT NOT NULL,
|
||||
description TEXT,
|
||||
createdAt DATETIME DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
`);
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS categories (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
formId INTEGER NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
sortOrder INTEGER DEFAULT 0,
|
||||
FOREIGN KEY (formId) REFERENCES forms (id) ON DELETE CASCADE
|
||||
);
|
||||
`);
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS fields (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
categoryId INTEGER NOT NULL,
|
||||
label TEXT NOT NULL,
|
||||
type TEXT NOT NULL CHECK(type IN ('text', 'number', 'date', 'textarea', 'boolean')),
|
||||
description TEXT,
|
||||
sortOrder INTEGER NOT NULL,
|
||||
FOREIGN KEY (categoryId) REFERENCES categories(id) ON DELETE CASCADE
|
||||
);
|
||||
`);
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS responses (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
formId INTEGER NOT NULL,
|
||||
submittedAt DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
FOREIGN KEY (formId) REFERENCES forms (id) ON DELETE CASCADE
|
||||
);
|
||||
`);
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS response_values (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
responseId INTEGER NOT NULL,
|
||||
fieldId INTEGER NOT NULL,
|
||||
value TEXT,
|
||||
FOREIGN KEY (responseId) REFERENCES responses (id) ON DELETE CASCADE,
|
||||
FOREIGN KEY (fieldId) REFERENCES fields (id) ON DELETE CASCADE
|
||||
);
|
||||
`);
|
||||
|
||||
console.log('Database tables ensured.');
|
||||
return db;
|
||||
} catch (err) {
|
||||
console.error('Error initializing database with better-sqlite3:', err.message);
|
||||
throw err; // Re-throw the error
|
||||
}
|
||||
}
|
||||
|
||||
export function getDb() {
|
||||
if (!db) {
|
||||
// Try to initialize if not already done (e.g., during hot reload)
|
||||
try {
|
||||
initializeDatabase();
|
||||
} catch (err) {
|
||||
throw new Error('Database not initialized and initialization failed.');
|
||||
}
|
||||
if (!db) { // Check again after trying to initialize
|
||||
throw new Error('Database not initialized. Call initializeDatabase first.');
|
||||
}
|
||||
}
|
||||
return db;
|
||||
}
|
||||
|
||||
// Optional: Add a function to close the database gracefully on server shutdown
|
||||
export function closeDatabase() {
|
||||
if (db) {
|
||||
db.close();
|
||||
db = null;
|
||||
console.log('Database connection closed.');
|
||||
}
|
||||
}
|
||||
// --- Settings Functions removed ---
|
||||
// Settings can now be accessed via prisma.setting.findUnique, prisma.setting.upsert, etc.
|
||||
|
|
File diff suppressed because it is too large
Load diff
|
@ -19,9 +19,10 @@ import {
|
|||
defineSsrRenderPreloadTag
|
||||
} from '#q-app/wrappers'
|
||||
|
||||
// Import database initialization and close function
|
||||
import { initializeDatabase, closeDatabase } from './database.js';
|
||||
import prisma from './database.js'; // Import the prisma client instance
|
||||
import apiRoutes from './routes/api.js';
|
||||
import cron from 'node-cron';
|
||||
import { generateAndStoreMantisSummary } from './services/mantisSummarizer.js';
|
||||
|
||||
/**
|
||||
* Create your webserver and return its instance.
|
||||
|
@ -35,12 +36,31 @@ export const create = defineSsrCreate((/* { ... } */) => {
|
|||
|
||||
// Initialize the database (now synchronous)
|
||||
try {
|
||||
initializeDatabase();
|
||||
console.log('Database initialized successfully.');
|
||||
console.log('Prisma Client is ready.'); // Log Prisma readiness
|
||||
|
||||
// Schedule the Mantis summary task after DB initialization
|
||||
// Run daily at 1:00 AM server time (adjust as needed)
|
||||
cron.schedule('0 1 * * *', async () => {
|
||||
console.log('Running scheduled Mantis summary task...');
|
||||
try {
|
||||
await generateAndStoreMantisSummary();
|
||||
console.log('Scheduled Mantis summary task completed.');
|
||||
} catch (error) {
|
||||
console.error('Error running scheduled Mantis summary task:', error);
|
||||
}
|
||||
}, {
|
||||
scheduled: true,
|
||||
timezone: "Europe/London" // Example: Set to your server's timezone
|
||||
});
|
||||
console.log('Mantis summary cron job scheduled.');
|
||||
|
||||
// Optional: Run once immediately on server start if needed
|
||||
generateAndStoreMantisSummary().catch(err => console.error('Initial Mantis summary failed:', err));
|
||||
|
||||
} catch (error) {
|
||||
console.error('Failed to initialize database:', error);
|
||||
console.error('Error during server setup:', error);
|
||||
// Optionally handle the error more gracefully, e.g., prevent server start
|
||||
process.exit(1); // Exit if DB connection fails
|
||||
process.exit(1); // Exit if setup fails
|
||||
}
|
||||
|
||||
// attackers can use this header to detect apps running Express
|
||||
|
@ -91,9 +111,14 @@ export const listen = defineSsrListen(({ app, devHttpsApp, port }) => {
|
|||
*
|
||||
* Can be async: defineSsrClose(async ({ ... }) => { ... })
|
||||
*/
|
||||
export const close = defineSsrClose(({ listenResult }) => {
|
||||
export const close = defineSsrClose(async ({ listenResult }) => {
|
||||
// Close the database connection when the server shuts down
|
||||
closeDatabase();
|
||||
try {
|
||||
await prisma.$disconnect();
|
||||
console.log('Prisma Client disconnected.');
|
||||
} catch (e) {
|
||||
console.error('Error disconnecting Prisma Client:', e);
|
||||
}
|
||||
|
||||
return listenResult.close()
|
||||
})
|
||||
|
|
199
src-ssr/services/emailSummarizer.js
Normal file
199
src-ssr/services/emailSummarizer.js
Normal file
|
@ -0,0 +1,199 @@
|
|||
import Imap from 'node-imap';
|
||||
import { simpleParser } from 'mailparser';
|
||||
import { GoogleGenAI } from '@google/genai';
|
||||
import prisma from '../database.js';
|
||||
|
||||
// --- Environment Variables ---
|
||||
const { GOOGLE_API_KEY } = process.env; // Added
|
||||
|
||||
// --- AI Setup ---
|
||||
const ai = GOOGLE_API_KEY ? new GoogleGenAI({
|
||||
apiKey: GOOGLE_API_KEY,
|
||||
}) : null; // Added
|
||||
|
||||
export async function fetchAndFormatEmails() {
|
||||
return new Promise((resolve, reject) => {
|
||||
const imapConfig = {
|
||||
user: process.env.OUTLOOK_EMAIL_ADDRESS,
|
||||
password: process.env.OUTLOOK_APP_PASSWORD,
|
||||
host: 'outlook.office365.com',
|
||||
port: 993,
|
||||
tls: true,
|
||||
tlsOptions: { rejectUnauthorized: false } // Adjust as needed for your environment
|
||||
};
|
||||
|
||||
const imap = new Imap(imapConfig);
|
||||
const emailsJson = [];
|
||||
|
||||
function openInbox(cb) {
|
||||
// Note: IMAP uses '/' as hierarchy separator, adjust if your server uses something else
|
||||
imap.openBox('SLSNotifications/Reports/Backups', false, cb);
|
||||
}
|
||||
|
||||
imap.once('ready', () => {
|
||||
openInbox((err, box) => {
|
||||
if (err) {
|
||||
imap.end();
|
||||
return reject(new Error(`Error opening mailbox: ${err.message}`));
|
||||
}
|
||||
|
||||
const yesterday = new Date();
|
||||
yesterday.setDate(yesterday.getDate() - 1);
|
||||
const searchCriteria = [['SINCE', yesterday.toISOString().split('T')[0]]]; // Search since midnight yesterday
|
||||
const fetchOptions = { bodies: ['HEADER.FIELDS (SUBJECT DATE)', 'TEXT'], struct: true };
|
||||
|
||||
imap.search(searchCriteria, (searchErr, results) => {
|
||||
if (searchErr) {
|
||||
imap.end();
|
||||
return reject(new Error(`Error searching emails: ${searchErr.message}`));
|
||||
}
|
||||
|
||||
if (results.length === 0) {
|
||||
console.log('No emails found from the last 24 hours.');
|
||||
imap.end();
|
||||
return resolve([]);
|
||||
}
|
||||
|
||||
const f = imap.fetch(results, fetchOptions);
|
||||
let processedCount = 0;
|
||||
|
||||
f.on('message', (msg, seqno) => {
|
||||
let header = '';
|
||||
let body = '';
|
||||
|
||||
msg.on('body', (stream, info) => {
|
||||
let buffer = '';
|
||||
stream.on('data', (chunk) => {
|
||||
buffer += chunk.toString('utf8');
|
||||
});
|
||||
stream.once('end', () => {
|
||||
if (info.which === 'TEXT') {
|
||||
body = buffer;
|
||||
} else {
|
||||
// Assuming HEADER.FIELDS (SUBJECT DATE) comes as one chunk
|
||||
header = buffer;
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
msg.once('attributes', (attrs) => {
|
||||
// Attributes might contain date if not fetched via header
|
||||
});
|
||||
|
||||
msg.once('end', async () => {
|
||||
try {
|
||||
// Use mailparser to handle potential encoding issues and structure
|
||||
const mail = await simpleParser(`Subject: ${header.match(/Subject: (.*)/i)?.[1] || ''}\nDate: ${header.match(/Date: (.*)/i)?.[1] || ''}\n\n${body}`);
|
||||
emailsJson.push({
|
||||
title: mail.subject || 'No Subject',
|
||||
time: mail.date ? mail.date.toISOString() : 'No Date',
|
||||
body: mail.text || mail.html || 'No Body Content' // Prefer text, fallback to html, then empty
|
||||
});
|
||||
} catch (parseErr) {
|
||||
console.error(`Error parsing email seqno ${seqno}:`, parseErr);
|
||||
// Decide if you want to reject or just skip this email
|
||||
}
|
||||
|
||||
processedCount++;
|
||||
if (processedCount === results.length) {
|
||||
// This check might be slightly inaccurate if errors occur,
|
||||
// but it's a common pattern. Consider refining with promises.
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
f.once('error', (fetchErr) => {
|
||||
console.error('Fetch error: ' + fetchErr);
|
||||
// Don't reject here immediately, might still get some emails
|
||||
});
|
||||
|
||||
f.once('end', () => {
|
||||
console.log('Done fetching all messages!');
|
||||
imap.end();
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
imap.once('error', (err) => {
|
||||
reject(new Error(`IMAP Connection Error: ${err.message}`));
|
||||
});
|
||||
|
||||
imap.once('end', () => {
|
||||
console.log('IMAP Connection ended.');
|
||||
resolve(emailsJson); // Resolve with the collected emails
|
||||
});
|
||||
|
||||
imap.connect();
|
||||
});
|
||||
}
|
||||
|
||||
// --- Email Summary Logic (New Function) ---
|
||||
export async function generateAndStoreEmailSummary() {
|
||||
console.log('Attempting to generate and store Email summary...');
|
||||
if (!ai) {
|
||||
console.error('Google AI API key not configured. Skipping email summary generation.');
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
// Get the prompt from the database settings using Prisma
|
||||
const setting = await prisma.setting.findUnique({
|
||||
where: { key: 'emailPrompt' }, // Use 'emailPrompt' as the key
|
||||
select: { value: true }
|
||||
});
|
||||
const promptTemplate = setting?.value;
|
||||
|
||||
if (!promptTemplate) {
|
||||
console.error('Email prompt not found in database settings (key: emailPrompt). Skipping summary generation.');
|
||||
return;
|
||||
}
|
||||
|
||||
const emails = await fetchAndFormatEmails();
|
||||
|
||||
let summaryText;
|
||||
if (emails.length === 0) {
|
||||
summaryText = "No relevant emails found in the last 24 hours.";
|
||||
console.log('No recent emails found for summary.');
|
||||
} else {
|
||||
console.log(`Found ${emails.length} recent emails. Generating summary...`);
|
||||
// Replace placeholder in the prompt template
|
||||
// Ensure your prompt template uses $EMAIL_DATA
|
||||
let prompt = promptTemplate.replaceAll("$EMAIL_DATA", JSON.stringify(emails, null, 2));
|
||||
|
||||
// Call the AI model (adjust model name and config as needed)
|
||||
const response = await ai.models.generateContent({
|
||||
"model": "gemini-2.5-preview-04-17",
|
||||
"contents": prompt,
|
||||
config: {
|
||||
temperature: 0 // Adjust temperature as needed
|
||||
}
|
||||
});
|
||||
|
||||
summaryText = response.text;
|
||||
console.log('Email summary generated successfully by AI.');
|
||||
}
|
||||
|
||||
// Store the summary in the database using Prisma upsert
|
||||
const today = new Date();
|
||||
today.setUTCHours(0, 0, 0, 0); // Use UTC start of day for consistency
|
||||
|
||||
await prisma.emailSummary.upsert({
|
||||
where: { summaryDate: today },
|
||||
update: {
|
||||
summaryText: summaryText,
|
||||
// generatedAt is updated automatically by @default(now())
|
||||
},
|
||||
create: {
|
||||
summaryDate: today,
|
||||
summaryText: summaryText,
|
||||
},
|
||||
});
|
||||
console.log(`Email summary for ${today.toISOString().split('T')[0]} stored/updated in the database.`);
|
||||
|
||||
} catch (error) {
|
||||
console.error("Error during Email summary generation/storage:", error);
|
||||
// Re-throw or handle as appropriate for your application
|
||||
throw error;
|
||||
}
|
||||
}
|
171
src-ssr/services/mantisSummarizer.js
Normal file
171
src-ssr/services/mantisSummarizer.js
Normal file
|
@ -0,0 +1,171 @@
|
|||
import axios from 'axios';
|
||||
import { GoogleGenAI } from '@google/genai';
|
||||
import prisma from '../database.js'; // Import Prisma client
|
||||
|
||||
// --- Environment Variables ---
|
||||
const {
|
||||
MANTIS_API_KEY,
|
||||
MANTIS_API_ENDPOINT,
|
||||
GOOGLE_API_KEY
|
||||
} = process.env;
|
||||
|
||||
// --- Mantis Summarizer Setup ---
|
||||
const ai = GOOGLE_API_KEY ? new GoogleGenAI({
|
||||
apiKey: GOOGLE_API_KEY,
|
||||
}) : null;
|
||||
|
||||
const usernameMap = {
|
||||
'credmore': 'Cameron Redmore',
|
||||
'dgibson': 'Dane Gibson',
|
||||
'egzibovskis': 'Ed Gzibovskis',
|
||||
'ascotney': 'Amanda Scotney',
|
||||
'gclough': 'Garry Clough',
|
||||
'slee': 'Sarah Lee',
|
||||
'dwalker': 'Dave Walker',
|
||||
'askaith': 'Amy Skaith',
|
||||
'dpotter': 'Danny Potter',
|
||||
'msmart': 'Michael Smart',
|
||||
// Add other usernames as needed
|
||||
};
|
||||
|
||||
async function getMantisTickets() {
|
||||
if (!MANTIS_API_ENDPOINT || !MANTIS_API_KEY) {
|
||||
throw new Error("Mantis API endpoint or key not configured in environment variables.");
|
||||
}
|
||||
const url = `${MANTIS_API_ENDPOINT}/issues?project_id=1&page_size=50&select=id,summary,description,created_at,updated_at,reporter,notes`;
|
||||
const headers = {
|
||||
'Authorization': `${MANTIS_API_KEY}`,
|
||||
'Accept': 'application/json',
|
||||
'Content-Type': 'application/json',
|
||||
};
|
||||
|
||||
try {
|
||||
const response = await axios.get(url, { headers });
|
||||
|
||||
const tickets = response.data.issues.filter((ticket) => {
|
||||
const ticketDate = new Date(ticket.updated_at);
|
||||
const thresholdDate = new Date();
|
||||
const currentDay = thresholdDate.getDay(); // Sunday = 0, Monday = 1, ...
|
||||
|
||||
// Go back 4 days if Monday (to include Fri, Sat, Sun), otherwise 2 days
|
||||
const daysToSubtract = currentDay === 1 ? 4 : 2;
|
||||
thresholdDate.setDate(thresholdDate.getDate() - daysToSubtract);
|
||||
thresholdDate.setHours(0, 0, 0, 0); // Start of the day
|
||||
|
||||
return ticketDate >= thresholdDate;
|
||||
}).map((ticket) => {
|
||||
return {
|
||||
id: ticket.id,
|
||||
summary: ticket.summary,
|
||||
description: ticket.description,
|
||||
created_at: ticket.created_at,
|
||||
updated_at: ticket.updated_at,
|
||||
reporter: usernameMap[ticket.reporter?.username] || ticket.reporter?.name || 'Unknown Reporter', // Safer access
|
||||
notes: (ticket.notes ? ticket.notes.filter((note) => {
|
||||
const noteDate = new Date(note.created_at);
|
||||
const thresholdDate = new Date();
|
||||
const currentDay = thresholdDate.getDay();
|
||||
const daysToSubtract = currentDay === 1 ? 4 : 2;
|
||||
thresholdDate.setDate(thresholdDate.getDate() - daysToSubtract);
|
||||
thresholdDate.setHours(0, 0, 0, 0); // Start of the day
|
||||
return noteDate >= thresholdDate;
|
||||
}) : []).map((note) => {
|
||||
const reporter = usernameMap[note.reporter?.username] || note.reporter?.name || 'Unknown Reporter'; // Safer access
|
||||
return {
|
||||
reporter,
|
||||
created_at: note.created_at,
|
||||
text: note.text,
|
||||
};
|
||||
}),
|
||||
};
|
||||
});
|
||||
|
||||
return tickets;
|
||||
} catch (error) {
|
||||
console.error("Error fetching Mantis tickets:", error.message);
|
||||
// Check if it's an Axios error and provide more details
|
||||
if (axios.isAxiosError(error)) {
|
||||
console.error("Axios error details:", error.response?.status, error.response?.data);
|
||||
throw new Error(`Failed to fetch Mantis tickets: ${error.response?.statusText || error.message}`);
|
||||
}
|
||||
throw new Error(`Failed to fetch Mantis tickets: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
// --- Mantis Summary Logic (Exported) --- //
|
||||
|
||||
export async function generateAndStoreMantisSummary() {
|
||||
console.log('Attempting to generate and store Mantis summary...');
|
||||
if (!ai) {
|
||||
console.error('Google AI API key not configured. Skipping summary generation.');
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
// Get the prompt from the database settings using Prisma
|
||||
const setting = await prisma.setting.findUnique({
|
||||
where: { key: 'mantisPrompt' },
|
||||
select: { value: true }
|
||||
});
|
||||
const promptTemplate = setting?.value;
|
||||
|
||||
if (!promptTemplate) {
|
||||
console.error('Mantis prompt not found in database settings (key: mantisPrompt). Skipping summary generation.');
|
||||
return;
|
||||
}
|
||||
|
||||
const tickets = await getMantisTickets();
|
||||
|
||||
let summaryText;
|
||||
if (tickets.length === 0) {
|
||||
summaryText = "No Mantis tickets updated recently.";
|
||||
console.log('No recent Mantis tickets found.');
|
||||
} else {
|
||||
console.log(`Found ${tickets.length} recent Mantis tickets. Generating summary...`);
|
||||
let prompt = promptTemplate.replaceAll("$DATE", new Date().toISOString().split('T')[0]);
|
||||
prompt = prompt.replaceAll("$MANTIS_TICKETS", JSON.stringify(tickets, null, 2));
|
||||
|
||||
const response = await ai.models.generateContent({
|
||||
"model": "gemini-2.5-flash-preview-04-17",
|
||||
"contents": prompt,
|
||||
config: {
|
||||
temperature: 0
|
||||
}
|
||||
});
|
||||
|
||||
summaryText = response.text;
|
||||
console.log('Mantis summary generated successfully by AI.');
|
||||
}
|
||||
|
||||
// Store the summary in the database using Prisma upsert
|
||||
const today = new Date();
|
||||
today.setUTCHours(0, 0, 0, 0); // Use UTC start of day for consistency
|
||||
|
||||
await prisma.mantisSummary.upsert({
|
||||
where: { summaryDate: today },
|
||||
update: {
|
||||
summaryText: summaryText,
|
||||
// generatedAt is updated automatically by @default(now())
|
||||
},
|
||||
create: {
|
||||
summaryDate: today,
|
||||
summaryText: summaryText,
|
||||
},
|
||||
});
|
||||
console.log(`Mantis summary for ${today.toISOString().split('T')[0]} stored/updated in the database.`);
|
||||
|
||||
} catch (error) {
|
||||
console.error("Error during Mantis summary generation/storage:", error);
|
||||
}
|
||||
}
|
||||
|
||||
export async function generateTodaysSummary() {
|
||||
console.log('Triggering Mantis summary generation via generateTodaysSummary...');
|
||||
try {
|
||||
await generateAndStoreMantisSummary();
|
||||
return { success: true, message: 'Summary generation process initiated.' };
|
||||
} catch (error) {
|
||||
console.error('Error occurred within generateTodaysSummary while calling generateAndStoreMantisSummary:', error);
|
||||
throw new Error('Failed to initiate Mantis summary generation.');
|
||||
}
|
||||
}
|
Loading…
Add table
Add a link
Reference in a new issue