diff --git a/package.json b/package.json index 0d03db3..5c9897b 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,8 @@ "build": "tsc", "start": "node dist/server.js", "dev": "tsx watch src/server.ts", + "scraper": "node dist/scraper.js", + "scraper:dev": "tsx watch src/scraper.ts", "test": "jest", "test:watch": "jest --watch", "lint": "eslint src/**/*.ts", diff --git a/src/scraper.ts b/src/scraper.ts new file mode 100644 index 0000000..9748d9c --- /dev/null +++ b/src/scraper.ts @@ -0,0 +1,61 @@ +import { ScrapingScheduler } from './services/ScrapingScheduler.js'; +import { FeedRepository } from './repositories/FeedRepository.js'; +import { DatabaseConnection } from './config/database.js'; +import { Logger } from './utils/logger.js'; + +let scheduler: ScrapingScheduler; + +async function initializeScraper() { + try { + // Connect to database + await DatabaseConnection.getInstance().connect(); + Logger.database.connected(); + + // Initialize repository and scheduler + const feedRepository = new FeedRepository(); + scheduler = new ScrapingScheduler(feedRepository, { + intervalMinutes: 30, // Run every 30 minutes + maxRetries: 2, + retryDelayMinutes: 5, + enabled: true + }); + + // Start the scheduler + scheduler.start(); + Logger.info('Scraping scheduler started successfully'); + + // Log initial stats + const stats = scheduler.getStats(); + Logger.info('Initial scheduler stats', stats); + + } catch (error) { + Logger.error('Failed to start scraper', { error }); + process.exit(1); + } +} + +const shutdown = async () => { + try { + if (scheduler) { + await scheduler.shutdown(); + Logger.info('Scraping scheduler stopped'); + } + + await DatabaseConnection.getInstance().disconnect(); + Logger.database.disconnected(); + process.exit(0); + } catch (error) { + Logger.error('Error during scraper shutdown', { error }); + process.exit(1); + } +}; + +// Handle graceful shutdown +process.on('SIGINT', shutdown); +process.on('SIGTERM', shutdown); + +// Start the scraper +initializeScraper().catch(error => { + Logger.error('Failed to initialize scraper', { error }); + process.exit(1); +}); \ No newline at end of file