Merge pull request #164 from Cryptkeeper/bulk-pings

5.4.0 release preview

assets/js/app.js

@@ -62,8 +62,6 @@ export class App {
 
   initTasks () {
     this._taskIds.push(setInterval(this.sortController.sortServers, 5000))
-    this._taskIds.push(setInterval(this.updateGlobalStats, 1000))
-    this._taskIds.push(setInterval(this.percentageBar.redraw, 1000))
   }
 
   handleDisconnect () {
@@ -102,26 +100,34 @@ export class App {
       .reduce((sum, current) => sum + current, 0)
   }
 
-  addServer = (pings) => {
+  addServer = (serverId, payload, timestampPoints) => {
     // Even if the backend has never pinged the server, the frontend is promised a placeholder object.
     // result = undefined
     // error = defined with "Waiting" description
     // info = safely defined with configured data
-    const latestPing = pings[pings.length - 1]
-    const serverRegistration = this.serverRegistry.createServerRegistration(latestPing.serverId)
+    const serverRegistration = this.serverRegistry.createServerRegistration(serverId)
 
-    serverRegistration.initServerStatus(latestPing)
+    serverRegistration.initServerStatus(payload)
 
-    // Push the historical data into the graph
-    // This will trim and format the data so it is ready for the graph to render once init
-    serverRegistration.addGraphPoints(pings)
+    // playerCountHistory is only defined when the backend has previous ping data
+    // undefined playerCountHistory means this is a placeholder ping generated by the backend
+    if (typeof payload.playerCountHistory !== 'undefined') {
+      // Push the historical data into the graph
+      // This will trim and format the data so it is ready for the graph to render once init
+      serverRegistration.addGraphPoints(payload.playerCountHistory, timestampPoints)
+
+      // Set initial playerCount to the payload's value
+      // This will always exist since it is explicitly generated by the backend
+      // This is used for any post-add rendering of things like the percentageBar
+      serverRegistration.playerCount = payload.playerCount
+    }
 
     // Create the plot instance internally with the restructured and cleaned data
     serverRegistration.buildPlotInstance()
 
     // Handle the last known state (if any) as an incoming update
     // This triggers the main update pipeline and enables centralized update handling
-    serverRegistration.updateServerStatus(latestPing, true, this.publicConfig.minecraftVersions)
+    serverRegistration.updateServerStatus(payload, this.publicConfig.minecraftVersions)
 
     // Allow the ServerRegistration to bind any DOM events with app instance context
     serverRegistration.initEventListeners()

assets/js/graph.js

@@ -57,14 +57,15 @@ export class GraphDisplayManager {
       return
     }
 
-    // Trim any outdated entries by filtering the array into a new array
-    const startTimestamp = new Date().getTime()
-    const newGraphData = this._graphData[serverId].filter(point => startTimestamp - point[0] <= this._app.publicConfig.graphDuration)
+    const graphData = this._graphData[serverId]
 
     // Push the new data from the method call request
-    newGraphData.push([timestamp, playerCount])
+    graphData.push([timestamp, playerCount])
 
-    this._graphData[serverId] = newGraphData
+    // Trim any outdated entries by filtering the array into a new array
+    if (graphData.length > this._app.publicConfig.graphMaxLength) {
+      graphData.shift()
+    }
   }
 
   loadLocalStorage () {
@@ -160,18 +161,6 @@ export class GraphDisplayManager {
     document.getElementById('settings-toggle').style.display = 'inline-block'
   }
 
-  // requestRedraw allows usages to request a redraw that may be performed, or cancelled, sometime later
-  // This allows multiple rapid, but individual updates, to clump into a single redraw instead
-  requestRedraw () {
-    if (this._redrawRequestTimeout) {
-      clearTimeout(this._redrawRequestTimeout)
-    }
-
-    // Schedule new delayed redraw call
-    // This can be cancelled by #requestRedraw, #redraw and #reset
-    this._redrawRequestTimeout = setTimeout(this.redraw, 1000)
-  }
-
   redraw = () => {
     // Use drawing as a hint to update settings
     // This may cause unnessecary localStorage updates, but its a rare and harmless outcome
@@ -182,14 +171,6 @@ export class GraphDisplayManager {
     this._plotInstance.setData(this.getVisibleGraphData())
     this._plotInstance.setupGrid()
     this._plotInstance.draw()
-
-    // undefine value so #clearTimeout is not called
-    // This is safe even if #redraw is manually called since it removes the pending work
-    if (this._redrawRequestTimeout) {
-      clearTimeout(this._redrawRequestTimeout)
-    }
-
-    this._redrawRequestTimeout = undefined
   }
 
   requestResize () {
@@ -347,12 +328,6 @@ export class GraphDisplayManager {
       this._resizeRequestTimeout = undefined
     }
 
-    if (this._redrawRequestTimeout) {
-      clearTimeout(this._redrawRequestTimeout)
-
-      this._redrawRequestTimeout = undefined
-    }
-
     // Reset modified DOM structures
     document.getElementById('big-graph-checkboxes').innerHTML = ''
     document.getElementById('big-graph-controls').style.display = 'none'

assets/js/servers.js

@@ -76,8 +76,6 @@ export class ServerRegistry {
   }
 }
 
-const SERVER_GRAPH_DATA_MAX_LENGTH = 72
-
 export class ServerRegistration {
   playerCount = 0
   isVisible = true
@@ -94,42 +92,33 @@ export class ServerRegistration {
     this._failedSequentialPings = 0
   }
 
-  addGraphPoints (points) {
-    // Test if the first point contains error.placeholder === true
-    // This is sent by the backend when the server hasn't been pinged yet
-    // These points will be disregarded to prevent the graph starting at 0 player count
-    points = points.filter(point => !point.error || !point.error.placeholder)
-
-    // The backend should never return more data elements than the max
-    // but trim the data result regardless for safety and performance purposes
-    if (points.length > SERVER_GRAPH_DATA_MAX_LENGTH) {
-      points.slice(points.length - SERVER_GRAPH_DATA_MAX_LENGTH, points.length)
+  addGraphPoints (points, timestampPoints) {
+    for (let i = 0; i < points.length; i++) {
+      const point = points[i]
+      const timestamp = timestampPoints[i]
+      this._graphData.push([timestamp, point])
     }
-
-    this._graphData = points.map(point => point.result ? [point.timestamp, point.result.players.online] : [point.timestamp, 0])
   }
 
   buildPlotInstance () {
     this._plotInstance = $.plot('#chart_' + this.serverId, [this._graphData], SERVER_GRAPH_OPTIONS)
   }
 
-  handlePing (payload, pushToGraph) {
-    if (payload.result) {
-      this.playerCount = payload.result.players.online
+  handlePing (payload, timestamp) {
+    if (typeof payload.playerCount !== 'undefined') {
+      this.playerCount = payload.playerCount
 
-      if (pushToGraph) {
-        // Only update graph for successful pings
-        // This intentionally pauses the server graph when pings begin to fail
-        this._graphData.push([payload.timestamp, this.playerCount])
+      // Only update graph for successful pings
+      // This intentionally pauses the server graph when pings begin to fail
+      this._graphData.push([timestamp, this.playerCount])
 
-        // Trim graphData to within the max length by shifting out the leading elements
-        if (this._graphData.length > SERVER_GRAPH_DATA_MAX_LENGTH) {
-          this._graphData.shift()
-        }
-
-        this.redraw()
+      // Trim graphData to within the max length by shifting out the leading elements
+      if (this._graphData.length > this._app.publicConfig.serverGraphMaxLength) {
+        this._graphData.shift()
       }
 
+      this.redraw()
+
       // Reset failed ping counter to ensure the next connection error
       // doesn't instantly retrigger a layout change
       this._failedSequentialPings = 0
@@ -169,11 +158,7 @@ export class ServerRegistration {
     this.lastPeakData = data
   }
 
-  updateServerStatus (ping, isInitialUpdate, minecraftVersions) {
-    // Only pushToGraph when initialUpdate === false
-    // Otherwise the ping value is pushed into the graphData when already present
-    this.handlePing(ping, !isInitialUpdate)
-
+  updateServerStatus (ping, minecraftVersions) {
     if (ping.versions) {
       const versionsElement = document.getElementById('version_' + this.serverId)
 
@@ -215,24 +200,27 @@ export class ServerRegistration {
       errorElement.style.display = 'block'
 
       errorElement.innerText = ping.error.message
-    } else if (ping.result) {
+    } else if (typeof ping.playerCount !== 'undefined') {
       // Ensure the player-count element is visible and hide the error element
       playerCountLabelElement.style.display = 'block'
       errorElement.style.display = 'none'
 
-      document.getElementById('player-count-value_' + this.serverId).innerText = formatNumber(ping.result.players.online)
+      document.getElementById('player-count-value_' + this.serverId).innerText = formatNumber(ping.playerCount)
+    }
+
+    // An updated favicon has been sent, update the src
+    if (ping.favicon) {
+      const faviconElement = document.getElementById('favicon_' + this.serverId)
 
-      // An updated favicon has been sent, update the src
-      // Ignore calls from 'add' events since they will have explicitly manually handled the favicon update
-      if (!isInitialUpdate && ping.favicon) {
-        document.getElementById('favicon_' + this.serverId).setAttribute('src', ping.favicon)
+      // Since favicons may be URLs, only update the attribute when it has changed
+      // Otherwise the browser may send multiple requests to the same URL
+      if (faviconElement.getAttribute('src') !== ping.favicon) {
+        faviconElement.setAttribute('src', ping.favicon)
       }
     }
   }
 
   initServerStatus (latestPing) {
-    const peakHourDuration = Math.floor(this._app.publicConfig.graphDuration / (60 * 60 * 1000)) + 'h Peak: '
-
     const serverElement = document.createElement('div')
 
     serverElement.id = 'container_' + this.serverId
@@ -244,7 +232,7 @@ export class ServerRegistration {
         '<h3 class="server-name"><span class="' + this._app.favoritesManager.getIconClass(this.isFavorite) + '" id="favorite-toggle_' + this.serverId + '"></span> ' + this.data.name + '</h3>' +
         '<span class="server-error" id="error_' + this.serverId + '"></span>' +
         '<span class="server-label" id="player-count_' + this.serverId + '">Players: <span class="server-value" id="player-count-value_' + this.serverId + '"></span></span>' +
-        '<span class="server-label" id="peak_' + this.serverId + '">' + peakHourDuration + '<span class="server-value" id="peak-value_' + this.serverId + '">-</span></span>' +
+        '<span class="server-label" id="peak_' + this.serverId + '">' + this._app.publicConfig.graphDurationLabel + ' Peak: <span class="server-value" id="peak-value_' + this.serverId + '">-</span></span>' +
         '<span class="server-label" id="record_' + this.serverId + '">Record: <span class="server-value" id="record-value_' + this.serverId + '">-</span></span>' +
         '<span class="server-label" id="version_' + this.serverId + '"></span>' +
       '</div>' +

assets/js/socket.js

@@ -67,7 +67,9 @@ export class SocketManager {
             }
           }
 
-          payload.servers.forEach(this._app.addServer)
+          payload.servers.forEach((serverPayload, serverId) => {
+            this._app.addServer(serverId, serverPayload, payload.timestampPoints)
+          })
 
           if (payload.mojangServices) {
             this._app.mojangUpdater.updateStatus(payload.mojangServices)
@@ -79,29 +81,45 @@ export class SocketManager {
 
           break
 
-        case 'updateServer': {
-          // The backend may send "update" events prior to receiving all "add" events
-          // A server has only been added once it's ServerRegistration is defined
-          // Checking undefined protects from this race condition
-          const serverRegistration = this._app.serverRegistry.getServerRegistration(payload.serverId)
+        case 'updateServers': {
+          let requestGraphRedraw = false
 
-          if (serverRegistration) {
-            serverRegistration.updateServerStatus(payload, false, this._app.publicConfig.minecraftVersions)
-          }
+          for (let serverId = 0; serverId < payload.updates.length; serverId++) {
+            // The backend may send "update" events prior to receiving all "add" events
+            // A server has only been added once it's ServerRegistration is defined
+            // Checking undefined protects from this race condition
+            const serverRegistration = this._app.serverRegistry.getServerRegistration(serverId)
+            const serverUpdate = payload.updates[serverId]
+
+            if (serverRegistration) {
+              serverRegistration.handlePing(serverUpdate, payload.timestamp)
 
-          // Use update payloads to conditionally append data to graph
-          // Skip any incoming updates if the graph is disabled
-          if (payload.updateHistoryGraph && this._app.graphDisplayManager.isVisible) {
-            // Update may not be successful, safely append 0 points
-            const playerCount = payload.result ? payload.result.players.online : 0
+              serverRegistration.updateServerStatus(serverUpdate, this._app.publicConfig.minecraftVersions)
+            }
 
-            this._app.graphDisplayManager.addGraphPoint(serverRegistration.serverId, payload.timestamp, playerCount)
+            // Use update payloads to conditionally append data to graph
+            // Skip any incoming updates if the graph is disabled
+            if (serverUpdate.updateHistoryGraph && this._app.graphDisplayManager.isVisible) {
+              // Update may not be successful, safely append 0 points
+              const playerCount = serverUpdate.playerCount || 0
 
-            // Only redraw the graph if not mutating hidden data
-            if (serverRegistration.isVisible) {
-              this._app.graphDisplayManager.requestRedraw()
+              this._app.graphDisplayManager.addGraphPoint(serverRegistration.serverId, payload.timestamp, playerCount)
+
+              // Only redraw the graph if not mutating hidden data
+              if (serverRegistration.isVisible) {
+                requestGraphRedraw = true
+              }
             }
           }
+
+          // Run redraw tasks after handling bulk updates
+          if (requestGraphRedraw) {
+            this._app.graphDisplayManager.redraw()
+          }
+
+          this._app.percentageBar.redraw()
+          this._app.updateGlobalStats()
+
           break
         }
 

assets/js/sort.js

@@ -8,7 +8,7 @@ const SORT_OPTIONS = [
   },
   {
     getName: (app) => {
-      return Math.floor(app.publicConfig.graphDuration / (60 * 60 * 1000)) + 'h Peak'
+      return app.publicConfig.graphDurationLabel + ' Peak'
     },
     sortFunc: (a, b) => {
       if (!a.lastPeakData && !b.lastPeakData) {

config.json

@@ -9,6 +9,11 @@
 		"pingAll": 3000,
 		"connectTimeout": 2500
 	},
-	"logToDatabase": false,
-	"graphDuration": 86400000
+	"performance": {
+		"skipUnfurlSrv": false,
+		"unfurlSrvCacheTtl": 120000
+	},
+	"logToDatabase": true,
+	"graphDuration": 86400000,
+	"serverGraphDuration": 180000
 }

docs/CHANGELOG.md

@@ -1,3 +1,13 @@
+**5.4.0** *(May 9 2020)*
+- Favicons are now served over the http server (using a unique hash). This allows the favicons to be safely cached for long durations and still support dynamic updates.
+- Adds "graphDurationLabel" to `config.json` which allows you to manually modify the "24h Peak" label to a custom time duration.
+- Adds "serverGraphDuration" (default 3 minutes) to `config.json` which allows you to specify the max time duration for the individual server player count graphs.
+- Adds "performance.skipUnfurlSrv" (default false) to `config.json` which allows you to skip SRV unfurling when pinging. For those who aren't pinging servers that use SRV records, this should help speed up ping times.
+- Adds "performance.skipUnfurlSrv" (default 120 seconds) to `config.json` which allows you specify how long a SRV unfurl should be cached for. This prevents repeated, potentially slow lookups. Set to 0 to disable caching.
+- Ping timestamps are now shared between all server pings. This means less data transfer when loading or updating the page, less memory usage by the backend and frontend, and less hectic updates on the frontend.
+- Optimized several protocol level schemas to remove legacy format waste. Less bandwidth!
+- Fixes a bug where favicons may not be updated if the page is loaded prior to their initialization.
+
 **5.3.1** *(May 5 2020)*
 - Fixes Mojang service status indicators not updating after initial page load.
 

lib/app.js

@@ -2,6 +2,7 @@ const Database = require('./database')
 const MojangUpdater = require('./mojang')
 const PingController = require('./ping')
 const Server = require('./server')
+const TimeTracker = require('./time')
 const MessageOf = require('./message')
 
 const config = require('../config')
@@ -13,7 +14,8 @@ class App {
   constructor () {
     this.mojangUpdater = new MojangUpdater(this)
     this.pingController = new PingController(this)
-    this.server = new Server(this.handleClientConnection)
+    this.server = new Server(this)
+    this.timeTracker = new TimeTracker(this)
   }
 
   loadDatabase (callback) {
@@ -66,13 +68,16 @@ class App {
 
         // Send configuration data for rendering the page
         return {
-          graphDuration: config.graphDuration,
-          servers: this.serverRegistrations.map(serverRegistration => serverRegistration.data),
+          graphDurationLabel: config.graphDurationLabel || (Math.floor(config.graphDuration / (60 * 60 * 1000)) + 'h'),
+          graphMaxLength: TimeTracker.getMaxGraphDataLength(),
+          serverGraphMaxLength: TimeTracker.getMaxServerGraphDataLength(),
+          servers: this.serverRegistrations.map(serverRegistration => serverRegistration.getPublicData()),
           minecraftVersions: minecraftVersionNames,
           isGraphVisible: config.logToDatabase
         }
       })(),
       mojangServices: this.mojangUpdater.getLastUpdate(),
+      timestampPoints: this.timeTracker.getPoints(),
       servers: this.serverRegistrations.map(serverRegistration => serverRegistration.getPingHistory())
     }