diff --git a/fecher.py b/fecher.py deleted file mode 100644 index c2b2dfa..0000000 --- a/fecher.py +++ /dev/null @@ -1,122 +0,0 @@ -import requests -from bs4 import BeautifulSoup -from pymongo.mongo_client import MongoClient -from datetime import datetime, timedelta -import os -import logging -from http.server import BaseHTTPRequestHandler - -# Configure logging -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - -def get_users(soup): - users = {} - try: - rows = soup.find_all('tr')[1:] # Skip the header row - for row in rows: - columns = row.find_all('td') - if len(columns) == 4: - user_name = columns[1].text.strip() - solved_tasks = int(columns[2].text.strip()) - users[user_name] = solved_tasks - except Exception as e: - logger.error(f"Error parsing users: {str(e)}") - return users - -def fetch_cses_data(): - cookies = { - 'PHPSESSID': os.environ.get('CSES_PHPSESSID', 'b614b76259290f9aaccda2a2afdd428118304b9a'), - } - - headers = { - 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8', - 'Accept-Language': 'en-GB,en-US;q=0.9,en;q=0.8', - 'Connection': 'keep-alive', - 'Cookie': f'PHPSESSID={cookies["PHPSESSID"]}', - 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36', - } - - users = {} - try: - for page in [1, 2]: - response = requests.get(f'https://cses.fi/problemset/stats/friends/p/{page}', - cookies=cookies, - headers=headers, - timeout=10) - response.raise_for_status() - users.update(get_users(BeautifulSoup(response.text, "html.parser"))) - - return users - except Exception as e: - logger.error(f"Error fetching CSES data: {str(e)}") - return None - -def update_mongodb(users_data): - if not users_data: - return False - - try: - uri = os.environ.get('MONGODB_URI') - if not uri: - logger.error("MongoDB URI not found in environment variables") - return False - - client = MongoClient(uri) - db = client.get_database("leaderboard") - collection = db.get_collection("CSES") - - for user, tasks in users_data.items(): - document = collection.find_one({"username":user}); - todayDate = (datetime.today()).strftime("%d/%m/%Y") - if document : - streak = int(document["streak"]) - prevSolved = document['solved'].get((datetime.today() - timedelta(days=1)).strftime("%d/%m/%Y")) - currSolved = prevSolved - if prevSolved != None and prevSolved < tasks: - streak += int(todayDate != document['lastUpdate']) - currSolved = tasks - else : - streak = 0 - document['solved'][todayDate] = currSolved - collection.update_one({"username":user} , {"$set": {"solved": document['solved'] , "streak": streak , "questionSolved" : tasks , "lastUpdate" : (datetime.today()).strftime("%d/%m/%Y") }}) - else : - data = {"username":user,"solved":{todayDate : tasks},"streak" : 0 , "questionSolved" : tasks } - result = collection.insert_one(data) - client.close() - return True - except Exception as e: - logger.error(f"Error updating MongoDB: {str(e)}") - return False - -class handler(BaseHTTPRequestHandler): - def do_GET(self): - try: - # Check if this is a cron job request - is_cron = self.headers.get('X-Vercel-Cron') == 'true' - - if not is_cron and self.path != '/fetch': - self.send_response(404) - self.end_headers() - return - - users_data = fetch_cses_data() - success = update_mongodb(users_data) - - self.send_response(200 if success else 500) - self.send_header('Content-type', 'application/json') - self.end_headers() - self.wfile.write(bytes('{"status": "success"}' if success else '{"status": "error"}', "utf-8")) - - except Exception as e: - logger.error(f"Handler error: {str(e)}") - self.send_response(500) - self.send_header('Content-type', 'application/json') - self.end_headers() - self.wfile.write(bytes('{"status": "error"}', "utf-8")) - -# For local testing -if __name__ == "__main__": - users_data = fetch_cses_data() - success = update_mongodb(users_data) - print("Update successful" if success else "Update failed") diff --git a/fetcher.js b/fetcher.js new file mode 100644 index 0000000..0fd2102 --- /dev/null +++ b/fetcher.js @@ -0,0 +1,123 @@ +const axios = require('axios'); +const cheerio = require('cheerio'); +const mongoose = require('mongoose'); +const moment = require('moment'); + +// Function to extract users from HTML +function getUsers($) { + const users = {}; + $('tr').slice(1).each((_, row) => { + const columns = $(row).find('td'); + if (columns.length === 4) { + const userName = $(columns[1]).text().trim(); + const solvedTasks = parseInt($(columns[2]).text().trim()); + users[userName] = solvedTasks; + } + }); + return users; +} + +// Function to fetch CSES data +async function fetchCSESData() { + const cookies = { + PHPSESSID: process.env.CSES_PHPSESSID || 'b614b76259290f9aaccda2a2afdd428118304b9a' + }; + + const headers = { + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8', + 'Accept-Language': 'en-GB,en-US;q=0.9,en;q=0.8', + 'Connection': 'keep-alive', + 'Cookie': `PHPSESSID=${cookies.PHPSESSID}`, + 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36' + }; + + const users = {}; + try { + for (let page of [1, 2]) { + const response = await axios.get(`https://cses.fi/problemset/stats/friends/p/${page}`, { + headers, + timeout: 10000 + }); + const $ = cheerio.load(response.data); + Object.assign(users, getUsers($)); + } + return users; + } catch (error) { + console.error('Error fetching CSES data:', error.message); + return null; + } +} + +// Function to update MongoDB +async function updateMongoDB(users) { + if (!users) return false; + + try { + const collection = mongoose.connection.collection('CSES'); + const todayDate = moment().format('DD/MM/YYYY'); + const yesterdayDate = moment().subtract(1, 'days').format('DD/MM/YYYY'); + + for (const [user, tasks] of Object.entries(users)) { + const document = await collection.findOne({ username: user }); + + if (document) { + let streak = parseInt(document.streak || 0); + const prevSolved = document.solved?.[yesterdayDate]; + let currSolved = prevSolved; + + if (prevSolved !== undefined && prevSolved < tasks) { + streak += (todayDate !== document.lastUpdate ? 1 : 0); + currSolved = tasks; + } else { + streak = 0; + } + + document.solved = document.solved || {}; + document.solved[todayDate] = currSolved; + + await collection.updateOne( + { username: user }, + { + $set: { + solved: document.solved, + streak: streak, + questionSolved: tasks, + lastUpdate: todayDate + } + } + ); + } else { + const data = { + username: user, + solved: { [todayDate]: tasks }, + streak: 0, + questionSolved: tasks, + lastUpdate: todayDate + }; + await collection.insertOne(data); + } + } + return true; + } catch (error) { + console.error('Error updating MongoDB:', error.message); + return false; + } +} + +// Main function to fetch and update data +async function updateLeaderboard() { + console.log('Starting leaderboard update:', new Date().toISOString()); + try { + const users = await fetchCSESData(); + if (users) { + const success = await updateMongoDB(users); + console.log('Update completed:', success ? 'successful' : 'failed'); + } else { + console.log('No user data fetched'); + } + } catch (error) { + console.error('Error in updateLeaderboard:', error.message); + } +} + +module.exports = { updateLeaderboard }; diff --git a/index.js b/index.js index 4537862..09186c2 100644 --- a/index.js +++ b/index.js @@ -2,6 +2,8 @@ const express = require('express'); const mongoose = require("mongoose"); const moment = require('moment'); const path = require('path'); +const cron = require('node-cron'); +const { updateLeaderboard } = require('./fetcher'); moment().format(); @@ -51,12 +53,8 @@ app.get("/", async (req, res) => { const noOfDaysInWeek = 7; for (let index = 0; index < noOfDaysInWeek; index++) { - const reqDate = moment("2024-12-08", "YYYY-MM-DD") - .subtract(index, 'days') - .format('DD/MM/YYYY'); - const prevDate = moment("2024-12-08", "YYYY-MM-DD") - .subtract(index + 1, 'days') - .format('DD/MM/YYYY'); + const reqDate = moment().subtract(index, 'days').format('DD/MM/YYYY'); + const prevDate = moment().subtract(index + 1, 'days').format('DD/MM/YYYY'); if (userData.solved && userData.solved[reqDate] !== undefined && @@ -84,23 +82,53 @@ app.get("/", async (req, res) => { } }); +// Manual update endpoint (protected) +app.post("/update", async (req, res) => { + const apiKey = req.headers['x-api-key']; + if (apiKey !== process.env.API_KEY) { + return res.status(401).json({ error: "Unauthorized" }); + } + + try { + await updateLeaderboard(); + res.json({ status: "success" }); + } catch (error) { + console.error("Error in manual update:", error); + res.status(500).json({ error: "Update failed" }); + } +}); + // Error handling middleware app.use((err, req, res, next) => { console.error(err.stack); - res.status(500).json({ error: "Something broke!", details: err.message }); + res.status(500).send('Something broke!'); +}); + +// Schedule the update every 3 hours +cron.schedule('0 */3 * * *', async () => { + console.log('Running scheduled update'); + try { + await updateLeaderboard(); + } catch (error) { + console.error('Scheduled update failed:', error); + } }); // Start server const startServer = async () => { + await connectDB(); + + // Initial update on server start try { - await connectDB(); - app.listen(port, () => { - console.log(`Server running on http://localhost:${port}`); - }); + await updateLeaderboard(); + console.log('Initial update completed'); } catch (error) { - console.error("Server startup error:", error); - process.exit(1); + console.error('Initial update failed:', error); } + + app.listen(port, () => { + console.log(`Server is running on port ${port}`); + }); }; startServer(); \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 13aa231..46532fd 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,10 +9,13 @@ "version": "1.0.0", "license": "ISC", "dependencies": { + "axios": "^1.7.8", + "cheerio": "^1.0.0", "ejs": "^3.1.10", "express": "^4.21.1", "moment": "^2.30.1", - "mongoose": "^8.8.3" + "mongoose": "^8.8.3", + "node-cron": "^3.0.3" }, "devDependencies": { "nodemon": "^3.1.7" @@ -99,6 +102,23 @@ "integrity": "sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA==", "license": "MIT" }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", + "license": "MIT" + }, + "node_modules/axios": { + "version": "1.7.8", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.8.tgz", + "integrity": "sha512-Uu0wb7KNqK2t5K+YQyVCLM76prD5sRFjKHbJYCP1J7JFGEQ6nN7HWn9+04LAeiJ3ji54lgS/gZCH1oxyrf1SPw==", + "license": "MIT", + "dependencies": { + "follow-redirects": "^1.15.6", + "form-data": "^4.0.0", + "proxy-from-env": "^1.1.0" + } + }, "node_modules/balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", @@ -142,6 +162,12 @@ "npm": "1.2.8000 || >= 1.4.16" } }, + "node_modules/boolbase": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", + "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==", + "license": "ISC" + }, "node_modules/brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -239,6 +265,48 @@ "node": ">=8" } }, + "node_modules/cheerio": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0.tgz", + "integrity": "sha512-quS9HgjQpdaXOvsZz82Oz7uxtXiy6UIsIQcpBj7HRw2M63Skasm9qlDocAM7jNuaxdhpPU7c4kJN+gA5MCu4ww==", + "license": "MIT", + "dependencies": { + "cheerio-select": "^2.1.0", + "dom-serializer": "^2.0.0", + "domhandler": "^5.0.3", + "domutils": "^3.1.0", + "encoding-sniffer": "^0.2.0", + "htmlparser2": "^9.1.0", + "parse5": "^7.1.2", + "parse5-htmlparser2-tree-adapter": "^7.0.0", + "parse5-parser-stream": "^7.1.2", + "undici": "^6.19.5", + "whatwg-mimetype": "^4.0.0" + }, + "engines": { + "node": ">=18.17" + }, + "funding": { + "url": "https://github.com/cheeriojs/cheerio?sponsor=1" + } + }, + "node_modules/cheerio-select": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/cheerio-select/-/cheerio-select-2.1.0.tgz", + "integrity": "sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==", + "license": "BSD-2-Clause", + "dependencies": { + "boolbase": "^1.0.0", + "css-select": "^5.1.0", + "css-what": "^6.1.0", + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, "node_modules/chokidar": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.6.0.tgz", @@ -282,6 +350,18 @@ "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", "license": "MIT" }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "license": "MIT", + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, "node_modules/concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -324,6 +404,34 @@ "integrity": "sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ==", "license": "MIT" }, + "node_modules/css-select": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/css-select/-/css-select-5.1.0.tgz", + "integrity": "sha512-nwoRF1rvRRnnCqqY7updORDsuqKzqYJ28+oSMaJMMgOauh3fvwHqMS7EZpIPqK8GL+g9mKxF1vP/ZjSeNjEVHg==", + "license": "BSD-2-Clause", + "dependencies": { + "boolbase": "^1.0.0", + "css-what": "^6.1.0", + "domhandler": "^5.0.2", + "domutils": "^3.0.1", + "nth-check": "^2.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, + "node_modules/css-what": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/css-what/-/css-what-6.1.0.tgz", + "integrity": "sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw==", + "license": "BSD-2-Clause", + "engines": { + "node": ">= 6" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, "node_modules/debug": { "version": "2.6.9", "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", @@ -350,6 +458,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "license": "MIT", + "engines": { + "node": ">=0.4.0" + } + }, "node_modules/depd": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", @@ -369,6 +486,61 @@ "npm": "1.2.8000 || >= 1.4.16" } }, + "node_modules/dom-serializer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz", + "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==", + "license": "MIT", + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.2", + "entities": "^4.2.0" + }, + "funding": { + "url": "https://github.com/cheeriojs/dom-serializer?sponsor=1" + } + }, + "node_modules/domelementtype": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz", + "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "BSD-2-Clause" + }, + "node_modules/domhandler": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz", + "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==", + "license": "BSD-2-Clause", + "dependencies": { + "domelementtype": "^2.3.0" + }, + "engines": { + "node": ">= 4" + }, + "funding": { + "url": "https://github.com/fb55/domhandler?sponsor=1" + } + }, + "node_modules/domutils": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.1.0.tgz", + "integrity": "sha512-H78uMmQtI2AhgDJjWeQmHwJJ2bLPD3GMmO7Zja/ZZh84wkm+4ut+IUnUdRa8uCGX88DiVx1j6FRe1XfxEgjEZA==", + "license": "BSD-2-Clause", + "dependencies": { + "dom-serializer": "^2.0.0", + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3" + }, + "funding": { + "url": "https://github.com/fb55/domutils?sponsor=1" + } + }, "node_modules/ee-first": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", @@ -399,6 +571,43 @@ "node": ">= 0.8" } }, + "node_modules/encoding-sniffer": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/encoding-sniffer/-/encoding-sniffer-0.2.0.tgz", + "integrity": "sha512-ju7Wq1kg04I3HtiYIOrUrdfdDvkyO9s5XM8QAj/bN61Yo/Vb4vgJxy5vi4Yxk01gWHbrofpPtpxM8bKger9jhg==", + "license": "MIT", + "dependencies": { + "iconv-lite": "^0.6.3", + "whatwg-encoding": "^3.1.1" + }, + "funding": { + "url": "https://github.com/fb55/encoding-sniffer?sponsor=1" + } + }, + "node_modules/encoding-sniffer/node_modules/iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/entities": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", + "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, "node_modules/es-define-property": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.0.tgz", @@ -538,6 +747,40 @@ "node": ">= 0.8" } }, + "node_modules/follow-redirects": { + "version": "1.15.9", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.9.tgz", + "integrity": "sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "license": "MIT", + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, + "node_modules/form-data": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.1.tgz", + "integrity": "sha512-tzN8e4TX8+kkxGPK8D5u0FNmjPUjw3lwC9lSLxxoB/+GtsJG91CO8bSWy73APlgAZzZbXEYZJuxjkHH2w+Ezhw==", + "license": "MIT", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/forwarded": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", @@ -682,6 +925,25 @@ "node": ">= 0.4" } }, + "node_modules/htmlparser2": { + "version": "9.1.0", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-9.1.0.tgz", + "integrity": "sha512-5zfg6mHUoaer/97TxnGpxmbR7zJtPwIYFMZ/H5ucTlPZhKvtum05yiPK3Mgai3a0DyVxv7qYqoweaEd2nrYQzQ==", + "funding": [ + "https://github.com/fb55/htmlparser2?sponsor=1", + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "MIT", + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.1.0", + "entities": "^4.5.0" + } + }, "node_modules/http-errors": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz", @@ -1035,6 +1297,18 @@ "node": ">= 0.6" } }, + "node_modules/node-cron": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/node-cron/-/node-cron-3.0.3.tgz", + "integrity": "sha512-dOal67//nohNgYWb+nWmg5dkFdIwDm8EpeGYMekPMrngV3637lqnX0lbUcCtgibHTz6SEz7DAIjKvKDFYCnO1A==", + "license": "ISC", + "dependencies": { + "uuid": "8.3.2" + }, + "engines": { + "node": ">=6.0.0" + } + }, "node_modules/nodemon": { "version": "3.1.7", "resolved": "https://registry.npmjs.org/nodemon/-/nodemon-3.1.7.tgz", @@ -1099,6 +1373,18 @@ "node": ">=0.10.0" } }, + "node_modules/nth-check": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz", + "integrity": "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==", + "license": "BSD-2-Clause", + "dependencies": { + "boolbase": "^1.0.0" + }, + "funding": { + "url": "https://github.com/fb55/nth-check?sponsor=1" + } + }, "node_modules/object-inspect": { "version": "1.13.3", "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.3.tgz", @@ -1123,6 +1409,43 @@ "node": ">= 0.8" } }, + "node_modules/parse5": { + "version": "7.2.1", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.2.1.tgz", + "integrity": "sha512-BuBYQYlv1ckiPdQi/ohiivi9Sagc9JG+Ozs0r7b/0iK3sKmrb0b9FdWdBbOdx6hBCM/F9Ir82ofnBhtZOjCRPQ==", + "license": "MIT", + "dependencies": { + "entities": "^4.5.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, + "node_modules/parse5-htmlparser2-tree-adapter": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/parse5-htmlparser2-tree-adapter/-/parse5-htmlparser2-tree-adapter-7.1.0.tgz", + "integrity": "sha512-ruw5xyKs6lrpo9x9rCZqZZnIUntICjQAd0Wsmp396Ul9lN/h+ifgVV1x1gZHi8euej6wTfpqX8j+BFQxF0NS/g==", + "license": "MIT", + "dependencies": { + "domhandler": "^5.0.3", + "parse5": "^7.0.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, + "node_modules/parse5-parser-stream": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/parse5-parser-stream/-/parse5-parser-stream-7.1.2.tgz", + "integrity": "sha512-JyeQc9iwFLn5TbvvqACIF/VXG6abODeB3Fwmv/TGdLk2LfbWkaySGY72at4+Ty7EkPZj854u4CrICqNk2qIbow==", + "license": "MIT", + "dependencies": { + "parse5": "^7.0.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, "node_modules/parseurl": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", @@ -1164,6 +1487,12 @@ "node": ">= 0.10" } }, + "node_modules/proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==", + "license": "MIT" + }, "node_modules/pstree.remy": { "version": "1.1.8", "resolved": "https://registry.npmjs.org/pstree.remy/-/pstree.remy-1.1.8.tgz", @@ -1480,6 +1809,15 @@ "dev": true, "license": "MIT" }, + "node_modules/undici": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-6.21.0.tgz", + "integrity": "sha512-BUgJXc752Kou3oOIuU1i+yZZypyZRqNPW0vqoMPl8VaoalSfeR0D8/t4iAS3yirs79SSMTxTag+ZC86uswv+Cw==", + "license": "MIT", + "engines": { + "node": ">=18.17" + } + }, "node_modules/unpipe": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", @@ -1498,6 +1836,15 @@ "node": ">= 0.4.0" } }, + "node_modules/uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", + "license": "MIT", + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/vary": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", @@ -1516,6 +1863,39 @@ "node": ">=12" } }, + "node_modules/whatwg-encoding": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-3.1.1.tgz", + "integrity": "sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ==", + "license": "MIT", + "dependencies": { + "iconv-lite": "0.6.3" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/whatwg-encoding/node_modules/iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/whatwg-mimetype": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-4.0.0.tgz", + "integrity": "sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, "node_modules/whatwg-url": { "version": "13.0.0", "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-13.0.0.tgz", diff --git a/package.json b/package.json index 09a51fa..8fdb256 100644 --- a/package.json +++ b/package.json @@ -14,10 +14,13 @@ "license": "ISC", "description": "", "dependencies": { + "axios": "^1.7.8", + "cheerio": "^1.0.0", "ejs": "^3.1.10", "express": "^4.21.1", "moment": "^2.30.1", - "mongoose": "^8.8.3" + "mongoose": "^8.8.3", + "node-cron": "^3.0.3" }, "devDependencies": { "nodemon": "^3.1.7" diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 650d6bf..0000000 --- a/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -requests==2.31.0 -beautifulsoup4==4.12.2 -pymongo==4.6.1 -python-dotenv==1.0.0