Convert Python fetcher to Node.js with 3-hour cron job

fetcher.js

@@ -0,0 +1,123 @@
+const axios = require('axios');
+const cheerio = require('cheerio');
+const mongoose = require('mongoose');
+const moment = require('moment');
+
+// Function to extract users from HTML
+function getUsers($) {
+    const users = {};
+    $('tr').slice(1).each((_, row) => {
+        const columns = $(row).find('td');
+        if (columns.length === 4) {
+            const userName = $(columns[1]).text().trim();
+            const solvedTasks = parseInt($(columns[2]).text().trim());
+            users[userName] = solvedTasks;
+        }
+    });
+    return users;
+}
+
+// Function to fetch CSES data
+async function fetchCSESData() {
+    const cookies = {
+        PHPSESSID: process.env.CSES_PHPSESSID || 'b614b76259290f9aaccda2a2afdd428118304b9a'
+    };
+
+    const headers = {
+        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
+        'Accept-Language': 'en-GB,en-US;q=0.9,en;q=0.8',
+        'Connection': 'keep-alive',
+        'Cookie': `PHPSESSID=${cookies.PHPSESSID}`,
+        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36'
+    };
+
+    const users = {};
+    try {
+        for (let page of [1, 2]) {
+            const response = await axios.get(`https://cses.fi/problemset/stats/friends/p/${page}`, {
+                headers,
+                timeout: 10000
+            });
+            const $ = cheerio.load(response.data);
+            Object.assign(users, getUsers($));
+        }
+        return users;
+    } catch (error) {
+        console.error('Error fetching CSES data:', error.message);
+        return null;
+    }
+}
+
+// Function to update MongoDB
+async function updateMongoDB(users) {
+    if (!users) return false;
+
+    try {
+        const collection = mongoose.connection.collection('CSES');
+        const todayDate = moment().format('DD/MM/YYYY');
+        const yesterdayDate = moment().subtract(1, 'days').format('DD/MM/YYYY');
+
+        for (const [user, tasks] of Object.entries(users)) {
+            const document = await collection.findOne({ username: user });
+
+            if (document) {
+                let streak = parseInt(document.streak || 0);
+                const prevSolved = document.solved?.[yesterdayDate];
+                let currSolved = prevSolved;
+
+                if (prevSolved !== undefined && prevSolved < tasks) {
+                    streak += (todayDate !== document.lastUpdate ? 1 : 0);
+                    currSolved = tasks;
+                } else {
+                    streak = 0;
+                }
+
+                document.solved = document.solved || {};
+                document.solved[todayDate] = currSolved;
+
+                await collection.updateOne(
+                    { username: user },
+                    {
+                        $set: {
+                            solved: document.solved,
+                            streak: streak,
+                            questionSolved: tasks,
+                            lastUpdate: todayDate
+                        }
+                    }
+                );
+            } else {
+                const data = {
+                    username: user,
+                    solved: { [todayDate]: tasks },
+                    streak: 0,
+                    questionSolved: tasks,
+                    lastUpdate: todayDate
+                };
+                await collection.insertOne(data);
+            }
+        }
+        return true;
+    } catch (error) {
+        console.error('Error updating MongoDB:', error.message);
+        return false;
+    }
+}
+
+// Main function to fetch and update data
+async function updateLeaderboard() {
+    console.log('Starting leaderboard update:', new Date().toISOString());
+    try {
+        const users = await fetchCSESData();
+        if (users) {
+            const success = await updateMongoDB(users);
+            console.log('Update completed:', success ? 'successful' : 'failed');
+        } else {
+            console.log('No user data fetched');
+        }
+    } catch (error) {
+        console.error('Error in updateLeaderboard:', error.message);
+    }
+}
+
+module.exports = { updateLeaderboard };

index.js

@@ -2,6 +2,8 @@ const express = require('express');
 const mongoose = require("mongoose");
 const moment = require('moment');
 const path = require('path');
+const cron = require('node-cron');
+const { updateLeaderboard } = require('./fetcher');
 
 moment().format();
 
@@ -51,12 +53,8 @@ app.get("/", async (req, res) => {
             const noOfDaysInWeek = 7;
 
             for (let index = 0; index < noOfDaysInWeek; index++) {
-                const reqDate = moment("2024-12-08", "YYYY-MM-DD")
-                    .subtract(index, 'days')
-                    .format('DD/MM/YYYY');
-                const prevDate = moment("2024-12-08", "YYYY-MM-DD")
-                    .subtract(index + 1, 'days')
-                    .format('DD/MM/YYYY');
+                const reqDate = moment().subtract(index, 'days').format('DD/MM/YYYY');
+                const prevDate = moment().subtract(index + 1, 'days').format('DD/MM/YYYY');
 
                 if (userData.solved && 
                     userData.solved[reqDate] !== undefined && 
@@ -84,23 +82,53 @@ app.get("/", async (req, res) => {
     }
 });
 
+// Manual update endpoint (protected)
+app.post("/update", async (req, res) => {
+    const apiKey = req.headers['x-api-key'];
+    if (apiKey !== process.env.API_KEY) {
+        return res.status(401).json({ error: "Unauthorized" });
+    }
+
+    try {
+        await updateLeaderboard();
+        res.json({ status: "success" });
+    } catch (error) {
+        console.error("Error in manual update:", error);
+        res.status(500).json({ error: "Update failed" });
+    }
+});
+
 // Error handling middleware
 app.use((err, req, res, next) => {
     console.error(err.stack);
-    res.status(500).json({ error: "Something broke!", details: err.message });
+    res.status(500).send('Something broke!');
+});
+
+// Schedule the update every 3 hours
+cron.schedule('0 */3 * * *', async () => {
+    console.log('Running scheduled update');
+    try {
+        await updateLeaderboard();
+    } catch (error) {
+        console.error('Scheduled update failed:', error);
+    }
 });
 
 // Start server
 const startServer = async () => {
+    await connectDB();
+
+    // Initial update on server start
     try {
-        await connectDB();
-        app.listen(port, () => {
-            console.log(`Server running on http://localhost:${port}`);
-        });
+        await updateLeaderboard();
+        console.log('Initial update completed');
     } catch (error) {
-        console.error("Server startup error:", error);
-        process.exit(1);
+        console.error('Initial update failed:', error);
     }
+
+    app.listen(port, () => {
+        console.log(`Server is running on port ${port}`);
+    });
 };
 
 startServer();