diff --git a/client/.eslintrc.js b/client/.eslintrc.js index 9a7b525..10e9f16 100644 --- a/client/.eslintrc.js +++ b/client/.eslintrc.js @@ -80,6 +80,7 @@ module.exports = { "no-bitwise": "error", "no-caller": "error", "no-cond-assign": "error", + "no-console": "error", "no-empty": "error", "no-empty-function": "off", "no-eval": "error", diff --git a/client/package-lock.json b/client/package-lock.json index d0347d2..040b9e8 100644 --- a/client/package-lock.json +++ b/client/package-lock.json @@ -1798,6 +1798,24 @@ "resolved": "https://registry.npmjs.org/@types/parse-json/-/parse-json-4.0.0.tgz", "integrity": "sha512-//oorEZjL6sbPcKUaCdIGlIUeH26mgzimjBB77G6XRgnDl/L5wOnpyBGRe/Mmf5CVW3PwEBE1NjiMZ/ssFh4wA==" }, + "@types/pino": { + "version": "6.3.3", + "resolved": "https://registry.npmjs.org/@types/pino/-/pino-6.3.3.tgz", + "integrity": "sha512-YtT58N7Tt7B7f5B/upuq694p4eT4icM9TuhgYeKhm+dnF0Ahm7q5YJp1i7vC2mBMdWgH1IvOa2XK6rhUjBv0GQ==", + "requires": { + "@types/node": "*", + "@types/pino-std-serializers": "*", + "@types/sonic-boom": "*" + } + }, + "@types/pino-std-serializers": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/@types/pino-std-serializers/-/pino-std-serializers-2.4.1.tgz", + "integrity": "sha512-17XcksO47M24IVTVKPeAByWUd3Oez7EbIjXpSbzMPhXVzgjGtrOa49gKBwxH9hb8dKv58OelsWQ+A1G1l9S3wQ==", + "requires": { + "@types/node": "*" + } + }, "@types/prop-types": { "version": "15.7.3", "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.3.tgz", @@ -1852,6 +1870,14 @@ "@types/react-router": "*" } }, + "@types/sonic-boom": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/@types/sonic-boom/-/sonic-boom-0.7.0.tgz", + "integrity": "sha512-AfqR0fZMoUXUNwusgXKxcE9DPlHNDHQp6nKYUd4PSRpLobF5CCevSpyTEBcVZreqaWKCnGBr9KI1fHMTttoB7A==", + "requires": { + "@types/node": "*" + } + }, "@types/stack-utils": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/@types/stack-utils/-/stack-utils-1.0.1.tgz", @@ -2580,6 +2606,32 @@ "sprintf-js": "~1.0.2" } }, + "args": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/args/-/args-5.0.1.tgz", + "integrity": "sha512-1kqmFCFsPffavQFGt8OxJdIcETti99kySRUPMpOhaGjL6mRJn8HFU1OxKY5bMqfZKUwTQc1mZkAjmGYaVOHFtQ==", + "dev": true, + "requires": { + "camelcase": "5.0.0", + "chalk": "2.4.2", + "leven": "2.1.0", + "mri": "1.1.4" + }, + "dependencies": { + "camelcase": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.0.0.tgz", + "integrity": "sha512-faqwZqnWxbxn+F1d399ygeamQNy3lPp/H9H6rNrqYh4FSVCtcY+3cub1MxA8o9mDd55mM8Aghuu/kuyYA6VTsA==", + "dev": true + }, + "leven": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/leven/-/leven-2.1.0.tgz", + "integrity": "sha1-wuep93IJTe6dNCAq6KzORoeHVYA=", + "dev": true + } + } + }, "aria-query": { "version": "4.2.2", "resolved": "https://registry.npmjs.org/aria-query/-/aria-query-4.2.2.tgz", @@ -2788,6 +2840,11 @@ "resolved": "https://registry.npmjs.org/atob/-/atob-2.1.2.tgz", "integrity": "sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg==" }, + "atomic-sleep": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/atomic-sleep/-/atomic-sleep-1.0.0.tgz", + "integrity": "sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ==" + }, "autoprefixer": { "version": "9.8.6", "resolved": "https://registry.npmjs.org/autoprefixer/-/autoprefixer-9.8.6.tgz", @@ -4614,6 +4671,12 @@ } } }, + "dateformat": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/dateformat/-/dateformat-3.0.3.tgz", + "integrity": "sha512-jyCETtSl3VMZMWeRo7iY1FL19ges1t55hMo5yaam4Jrsm5EPL89UQkoQRyiI+Yf4k8r2ZpdngkV8hr1lIdjb3Q==", + "dev": true + }, "debug": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz", @@ -6026,6 +6089,16 @@ "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", "integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=" }, + "fast-redact": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/fast-redact/-/fast-redact-3.0.0.tgz", + "integrity": "sha512-a/S/Hp6aoIjx7EmugtzLqXmcNsyFszqbt6qQ99BdG61QjBZF6shNis0BYR6TsZOQ1twYc0FN2Xdhwwbv6+KD0w==" + }, + "fast-safe-stringify": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.0.7.tgz", + "integrity": "sha512-Utm6CdzT+6xsDk2m8S6uL8VHxNwI6Jub+e9NYTcAms28T84pTa25GJQV9j0CY0N1rM8hK4x6grpF2BQf+2qwVA==" + }, "fastq": { "version": "1.8.0", "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.8.0.tgz", @@ -6170,6 +6243,11 @@ "write": "1.0.3" } }, + "flatstr": { + "version": "1.0.12", + "resolved": "https://registry.npmjs.org/flatstr/-/flatstr-1.0.12.tgz", + "integrity": "sha512-4zPxDyhCyiN2wIAtSLI6gc82/EjqZc1onI4Mz/l0pWrAlsSfYH/2ZIcU+e3oA2wDwbzIWNKwa23F8rh6+DRWkw==" + }, "flatted": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/flatted/-/flatted-2.0.2.tgz", @@ -7991,6 +8069,18 @@ } } }, + "jmespath": { + "version": "0.15.0", + "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.15.0.tgz", + "integrity": "sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc=", + "dev": true + }, + "joycon": { + "version": "2.2.5", + "resolved": "https://registry.npmjs.org/joycon/-/joycon-2.2.5.tgz", + "integrity": "sha512-YqvUxoOcVPnCp0VU1/56f+iKSdvIRJYPznH22BdXV3xMk75SFXhWeJkZ8C9XxUWt1b5x2X1SxuFygW1U0FmkEQ==", + "dev": true + }, "js-tokens": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", @@ -8748,6 +8838,12 @@ "run-queue": "^1.0.3" } }, + "mri": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/mri/-/mri-1.1.4.tgz", + "integrity": "sha512-6y7IjGPm8AzlvoUrwAaw1tLnUBudaS3752vcd8JtrpGGQn+rXIe63LFVHm/YMwtqAuh+LJPCFdlLYPWM1nYn6w==", + "dev": true + }, "ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", @@ -9579,6 +9675,100 @@ "pinkie": "^2.0.0" } }, + "pino": { + "version": "6.7.0", + "resolved": "https://registry.npmjs.org/pino/-/pino-6.7.0.tgz", + "integrity": "sha512-vPXJ4P9rWCwzlTJt+f0Ni4THc3DWyt8iDDCO4edQ8narTu6hnpzdXu8FqeSJCGndl1W6lfbYQUQihUO54y66Lw==", + "requires": { + "fast-redact": "^3.0.0", + "fast-safe-stringify": "^2.0.7", + "flatstr": "^1.0.12", + "pino-std-serializers": "^2.4.2", + "quick-format-unescaped": "^4.0.1", + "sonic-boom": "^1.0.2" + } + }, + "pino-pretty": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/pino-pretty/-/pino-pretty-4.3.0.tgz", + "integrity": "sha512-uEc9SUCCGVEs0goZvyznKXBHtI1PNjGgqHviJHxOCEFEWZN6Z/IQKv5pO9gSdm/b+WfX+/dfheWhtZUyScqjlQ==", + "dev": true, + "requires": { + "@hapi/bourne": "^2.0.0", + "args": "^5.0.1", + "chalk": "^4.0.0", + "dateformat": "^3.0.3", + "fast-safe-stringify": "^2.0.7", + "jmespath": "^0.15.0", + "joycon": "^2.2.5", + "pump": "^3.0.0", + "readable-stream": "^3.6.0", + "split2": "^3.1.1", + "strip-json-comments": "^3.1.1" + }, + "dependencies": { + "@hapi/bourne": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@hapi/bourne/-/bourne-2.0.0.tgz", + "integrity": "sha512-WEezM1FWztfbzqIUbsDzFRVMxSoLy3HugVcux6KDDtTqzPsLE8NDRHfXvev66aH1i2oOKKar3/XDjbvh/OUBdg==", + "dev": true + }, + "ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "requires": { + "color-convert": "^2.0.1" + } + }, + "chalk": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.0.tgz", + "integrity": "sha512-qwx12AxXe2Q5xQ43Ac//I6v5aXTipYrSESdOgzrN+9XjgEpyjpKuvSGaN4qE93f7TQTlerQQ8S+EQ0EyDoVL1A==", + "dev": true, + "requires": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + } + }, + "color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "requires": { + "color-name": "~1.1.4" + } + }, + "color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true + }, + "has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true + }, + "supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "requires": { + "has-flag": "^4.0.0" + } + } + } + }, + "pino-std-serializers": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/pino-std-serializers/-/pino-std-serializers-2.5.0.tgz", + "integrity": "sha512-wXqbqSrIhE58TdrxxlfLwU9eDhrzppQDvGhBEr1gYbzzM4KKo3Y63gSjiDXRKLVS2UOXdPNR2v+KnQgNrs+xUg==" + }, "pirates": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/pirates/-/pirates-4.0.1.tgz", @@ -10752,6 +10942,11 @@ "resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.2.0.tgz", "integrity": "sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==" }, + "quick-format-unescaped": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/quick-format-unescaped/-/quick-format-unescaped-4.0.1.tgz", + "integrity": "sha512-RyYpQ6Q5/drsJyOhrWHYMWTedvjTIat+FTwv0K4yoUxzvekw2aRHMQJLlnvt8UantkZg2++bEzD9EdxXqkWf4A==" + }, "raf": { "version": "3.4.1", "resolved": "https://registry.npmjs.org/raf/-/raf-3.4.1.tgz", @@ -12312,6 +12507,15 @@ } } }, + "sonic-boom": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-1.3.0.tgz", + "integrity": "sha512-4nX6OYvOYr6R76xfQKi6cZpTO3YSWe/vd+QdIfoH0lBy0MnPkeAbb2rRWgmgADkXUeCKPwO1FZAKlAVWAadELw==", + "requires": { + "atomic-sleep": "^1.0.0", + "flatstr": "^1.0.12" + } + }, "sort-keys": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/sort-keys/-/sort-keys-1.1.2.tgz", @@ -12417,6 +12621,15 @@ "extend-shallow": "^3.0.0" } }, + "split2": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/split2/-/split2-3.2.2.tgz", + "integrity": "sha512-9NThjpgZnifTkJpzTZ7Eue85S49QwpNhZTq6GRJwObb6jnLFNGB7Qm73V5HewTROPyxD0C29xqmaI68bQtV+hg==", + "dev": true, + "requires": { + "readable-stream": "^3.0.0" + } + }, "sprintf-js": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", diff --git a/client/package.json b/client/package.json index 8fb68cb..10663e8 100644 --- a/client/package.json +++ b/client/package.json @@ -8,10 +8,12 @@ "@testing-library/user-event": "^7.1.2", "@types/jest": "^24.0.0", "@types/node": "^12.0.0", + "@types/pino": "^6.3.3", "@types/react": "^16.9.0", "@types/react-dom": "^16.9.0", "@types/react-router-dom": "^5.1.6", "@types/styled-components": "^5.1.4", + "pino": "^6.7.0", "react": "^16.13.1", "react-dom": "^16.13.1", "react-router-dom": "^5.2.0", @@ -45,6 +47,7 @@ "devDependencies": { "@typescript-eslint/eslint-plugin-tslint": "^4.5.0", "eslint": "^6.6.0", + "pino-pretty": "^4.3.0", "tslint": "^6.1.3", "tslint-microsoft-contrib": "^6.2.0" } diff --git a/client/src/logger.ts b/client/src/logger.ts new file mode 100644 index 0000000..66dcea5 --- /dev/null +++ b/client/src/logger.ts @@ -0,0 +1,22 @@ +import pino from "pino"; + +let environment = (process.env.NODE_ENV) || "development"; + +// Parent logger. If you want to make use of logging in a new file, you MUST import this +// and create a child logger with constructor argument {module: "modulename"} +const logger = pino({ + /** + * Sets the log level as LOG_LEVEL in env or defaults to "info" + * If set to "info", all logs EXCEPT for "trace" and "debug" are shown + * Change LOG_LEVEL to "trace" if you would like to see all log levels, + * or "debug" if you want to see everything but trace logs. + */ + level: process.env.LOG_LEVEL || "info", + /** + * Makes pretty (non-JSON) logs output to stdout. Should be disabled in + * production + */ + prettyPrint: (environment === "development") +}); + +export default logger; diff --git a/server/.eslintrc.js b/server/.eslintrc.js index 9a7b525..f1d75cf 100644 --- a/server/.eslintrc.js +++ b/server/.eslintrc.js @@ -21,12 +21,6 @@ module.exports = { "max-lines": "off", "max-nested-callbacks": "off", } - }, - { - "files": ["src/Util.ts"], - "rules": { - "no-console": "off" - } } ], "ignorePatterns": [ @@ -80,6 +74,7 @@ module.exports = { "no-bitwise": "error", "no-caller": "error", "no-cond-assign": "error", + "no-console": "error", "no-empty": "error", "no-empty-function": "off", "no-eval": "error", diff --git a/server/Dockerfile b/server/Dockerfile index f1d9b80..13ec364 100644 --- a/server/Dockerfile +++ b/server/Dockerfile @@ -2,6 +2,17 @@ FROM node:lts WORKDIR /app COPY ./package*.json ./ + +# This installs Chromium's dependencies, which is required by puppeteer to run +RUN apt-get update \ + && apt-get install -y wget gnupg \ + && wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - \ + && sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list' \ + && apt-get update \ + && apt-get install -y google-chrome-stable fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst fonts-freefont-ttf libxss1 \ + --no-install-recommends \ + && rm -rf /var/lib/apt/lists/* + RUN npm ci COPY . . diff --git a/server/package-lock.json b/server/package-lock.json index e566443..ddde7d0 100644 --- a/server/package-lock.json +++ b/server/package-lock.json @@ -108,6 +108,12 @@ } } }, + "@hapi/bourne": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@hapi/bourne/-/bourne-2.0.0.tgz", + "integrity": "sha512-WEezM1FWztfbzqIUbsDzFRVMxSoLy3HugVcux6KDDtTqzPsLE8NDRHfXvev66aH1i2oOKKar3/XDjbvh/OUBdg==", + "dev": true + }, "@nodelib/fs.scandir": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.3.tgz", @@ -183,6 +189,14 @@ "@types/serve-static": "*" } }, + "@types/express-pino-logger": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@types/express-pino-logger/-/express-pino-logger-4.0.2.tgz", + "integrity": "sha512-pjFbznTQ/h9Dht4YP1Zqfj2+4S8K4u7K+EowklXhf0aL43KUIYo126w2h8mtwoHfa91Z3BYA8w2wfUfwNAHroA==", + "requires": { + "@types/pino-http": "*" + } + }, "@types/express-serve-static-core": { "version": "4.17.13", "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.13.tgz", @@ -223,6 +237,32 @@ "resolved": "https://registry.npmjs.org/@types/pg-types/-/pg-types-1.11.5.tgz", "integrity": "sha512-L8ogeT6vDzT1vxlW3KITTCt+BVXXVkLXfZ/XNm6UqbcJgxf+KPO7yjWx7dQQE8RW07KopL10x2gNMs41+IkMGQ==" }, + "@types/pino": { + "version": "6.3.3", + "resolved": "https://registry.npmjs.org/@types/pino/-/pino-6.3.3.tgz", + "integrity": "sha512-YtT58N7Tt7B7f5B/upuq694p4eT4icM9TuhgYeKhm+dnF0Ahm7q5YJp1i7vC2mBMdWgH1IvOa2XK6rhUjBv0GQ==", + "requires": { + "@types/node": "*", + "@types/pino-std-serializers": "*", + "@types/sonic-boom": "*" + } + }, + "@types/pino-http": { + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/@types/pino-http/-/pino-http-5.0.5.tgz", + "integrity": "sha512-xx8bMUmap1LV0qxzZUlsjE9F5oX3nzoh+rPEtCY8WQgWx6UESbEv4VFwcRLRj6mtACBd0BVcTydcvBppnb6lSg==", + "requires": { + "@types/pino": "*" + } + }, + "@types/pino-std-serializers": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/@types/pino-std-serializers/-/pino-std-serializers-2.4.1.tgz", + "integrity": "sha512-17XcksO47M24IVTVKPeAByWUd3Oez7EbIjXpSbzMPhXVzgjGtrOa49gKBwxH9hb8dKv58OelsWQ+A1G1l9S3wQ==", + "requires": { + "@types/node": "*" + } + }, "@types/puppeteer": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/@types/puppeteer/-/puppeteer-3.0.2.tgz", @@ -250,6 +290,14 @@ "@types/mime": "*" } }, + "@types/sonic-boom": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/@types/sonic-boom/-/sonic-boom-0.7.0.tgz", + "integrity": "sha512-AfqR0fZMoUXUNwusgXKxcE9DPlHNDHQp6nKYUd4PSRpLobF5CCevSpyTEBcVZreqaWKCnGBr9KI1fHMTttoB7A==", + "requires": { + "@types/node": "*" + } + }, "@types/strip-bom": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/@types/strip-bom/-/strip-bom-3.0.0.tgz", @@ -512,6 +560,61 @@ "sprintf-js": "~1.0.2" } }, + "args": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/args/-/args-5.0.1.tgz", + "integrity": "sha512-1kqmFCFsPffavQFGt8OxJdIcETti99kySRUPMpOhaGjL6mRJn8HFU1OxKY5bMqfZKUwTQc1mZkAjmGYaVOHFtQ==", + "dev": true, + "requires": { + "camelcase": "5.0.0", + "chalk": "2.4.2", + "leven": "2.1.0", + "mri": "1.1.4" + }, + "dependencies": { + "ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dev": true, + "requires": { + "color-convert": "^1.9.0" + } + }, + "camelcase": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.0.0.tgz", + "integrity": "sha512-faqwZqnWxbxn+F1d399ygeamQNy3lPp/H9H6rNrqYh4FSVCtcY+3cub1MxA8o9mDd55mM8Aghuu/kuyYA6VTsA==", + "dev": true + }, + "chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dev": true, + "requires": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + } + }, + "color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dev": true, + "requires": { + "color-name": "1.1.3" + } + }, + "color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=", + "dev": true + } + } + }, "array-find-index": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/array-find-index/-/array-find-index-1.0.2.tgz", @@ -534,6 +637,11 @@ "integrity": "sha512-+Ryf6g3BKoRc7jfp7ad8tM4TtMiaWvbF/1/sQcZPkkS7ag3D5nMBCe2UfOTONtAkaG0tO0ij3C5Lwmf1EiyjHg==", "dev": true }, + "atomic-sleep": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/atomic-sleep/-/atomic-sleep-1.0.0.tgz", + "integrity": "sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ==" + }, "balanced-match": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", @@ -1309,6 +1417,14 @@ } } }, + "express-pino-logger": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/express-pino-logger/-/express-pino-logger-5.0.0.tgz", + "integrity": "sha512-pwbZ2E712evTuJfU/+svG/xdXMiS5Awjbnbg382ikQfgb/5bLwF1Auzv8QXVv/x+GixUlzfPbYTgVLhacvocKw==", + "requires": { + "pino-http": "^5.1.0" + } + }, "extract-zip": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-2.0.1.tgz", @@ -1370,6 +1486,31 @@ "integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=", "dev": true }, + "fast-redact": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/fast-redact/-/fast-redact-3.0.0.tgz", + "integrity": "sha512-a/S/Hp6aoIjx7EmugtzLqXmcNsyFszqbt6qQ99BdG61QjBZF6shNis0BYR6TsZOQ1twYc0FN2Xdhwwbv6+KD0w==" + }, + "fast-safe-stringify": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.0.7.tgz", + "integrity": "sha512-Utm6CdzT+6xsDk2m8S6uL8VHxNwI6Jub+e9NYTcAms28T84pTa25GJQV9j0CY0N1rM8hK4x6grpF2BQf+2qwVA==" + }, + "fast-url-parser": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/fast-url-parser/-/fast-url-parser-1.1.3.tgz", + "integrity": "sha1-9K8+qfNNiicc9YrSs3WfQx8LMY0=", + "requires": { + "punycode": "^1.3.2" + }, + "dependencies": { + "punycode": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", + "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4=" + } + } + }, "fastq": { "version": "1.8.0", "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.8.0.tgz", @@ -1464,6 +1605,11 @@ } } }, + "flatstr": { + "version": "1.0.12", + "resolved": "https://registry.npmjs.org/flatstr/-/flatstr-1.0.12.tgz", + "integrity": "sha512-4zPxDyhCyiN2wIAtSLI6gc82/EjqZc1onI4Mz/l0pWrAlsSfYH/2ZIcU+e3oA2wDwbzIWNKwa23F8rh6+DRWkw==" + }, "flatted": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/flatted/-/flatted-2.0.2.tgz", @@ -1818,6 +1964,18 @@ "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=", "dev": true }, + "jmespath": { + "version": "0.15.0", + "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.15.0.tgz", + "integrity": "sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc=", + "dev": true + }, + "joycon": { + "version": "2.2.5", + "resolved": "https://registry.npmjs.org/joycon/-/joycon-2.2.5.tgz", + "integrity": "sha512-YqvUxoOcVPnCp0VU1/56f+iKSdvIRJYPznH22BdXV3xMk75SFXhWeJkZ8C9XxUWt1b5x2X1SxuFygW1U0FmkEQ==", + "dev": true + }, "js-tokens": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", @@ -1867,6 +2025,12 @@ "package-json": "^6.3.0" } }, + "leven": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/leven/-/leven-2.1.0.tgz", + "integrity": "sha1-wuep93IJTe6dNCAq6KzORoeHVYA=", + "dev": true + }, "levn": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", @@ -2036,6 +2200,12 @@ "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==" }, + "mri": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/mri/-/mri-1.1.4.tgz", + "integrity": "sha512-6y7IjGPm8AzlvoUrwAaw1tLnUBudaS3752vcd8JtrpGGQn+rXIe63LFVHm/YMwtqAuh+LJPCFdlLYPWM1nYn6w==", + "dev": true + }, "ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", @@ -2325,6 +2495,92 @@ "pinkie": "^2.0.0" } }, + "pino": { + "version": "6.7.0", + "resolved": "https://registry.npmjs.org/pino/-/pino-6.7.0.tgz", + "integrity": "sha512-vPXJ4P9rWCwzlTJt+f0Ni4THc3DWyt8iDDCO4edQ8narTu6hnpzdXu8FqeSJCGndl1W6lfbYQUQihUO54y66Lw==", + "requires": { + "fast-redact": "^3.0.0", + "fast-safe-stringify": "^2.0.7", + "flatstr": "^1.0.12", + "pino-std-serializers": "^2.4.2", + "quick-format-unescaped": "^4.0.1", + "sonic-boom": "^1.0.2" + } + }, + "pino-http": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/pino-http/-/pino-http-5.3.0.tgz", + "integrity": "sha512-aV4e7L8ez2MCa1qsuuliKYX5CDJug3LjW8oht+r4H4+fcf7ZvxPOppTs7P9dNHccF8k8oqhOcU/myiP4GvzJiA==", + "requires": { + "fast-url-parser": "^1.1.3", + "pino": "^6.0.0", + "pino-std-serializers": "^2.4.0" + } + }, + "pino-pretty": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/pino-pretty/-/pino-pretty-4.3.0.tgz", + "integrity": "sha512-uEc9SUCCGVEs0goZvyznKXBHtI1PNjGgqHviJHxOCEFEWZN6Z/IQKv5pO9gSdm/b+WfX+/dfheWhtZUyScqjlQ==", + "dev": true, + "requires": { + "@hapi/bourne": "^2.0.0", + "args": "^5.0.1", + "chalk": "^4.0.0", + "dateformat": "^3.0.3", + "fast-safe-stringify": "^2.0.7", + "jmespath": "^0.15.0", + "joycon": "^2.2.5", + "pump": "^3.0.0", + "readable-stream": "^3.6.0", + "split2": "^3.1.1", + "strip-json-comments": "^3.1.1" + }, + "dependencies": { + "chalk": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.0.tgz", + "integrity": "sha512-qwx12AxXe2Q5xQ43Ac//I6v5aXTipYrSESdOgzrN+9XjgEpyjpKuvSGaN4qE93f7TQTlerQQ8S+EQ0EyDoVL1A==", + "dev": true, + "requires": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + } + }, + "dateformat": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/dateformat/-/dateformat-3.0.3.tgz", + "integrity": "sha512-jyCETtSl3VMZMWeRo7iY1FL19ges1t55hMo5yaam4Jrsm5EPL89UQkoQRyiI+Yf4k8r2ZpdngkV8hr1lIdjb3Q==", + "dev": true + }, + "has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true + }, + "strip-json-comments": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", + "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", + "dev": true + }, + "supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "requires": { + "has-flag": "^4.0.0" + } + } + } + }, + "pino-std-serializers": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/pino-std-serializers/-/pino-std-serializers-2.5.0.tgz", + "integrity": "sha512-wXqbqSrIhE58TdrxxlfLwU9eDhrzppQDvGhBEr1gYbzzM4KKo3Y63gSjiDXRKLVS2UOXdPNR2v+KnQgNrs+xUg==" + }, "pkg-dir": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz", @@ -2471,6 +2727,11 @@ "resolved": "https://registry.npmjs.org/qs/-/qs-6.7.0.tgz", "integrity": "sha512-VCdBRNFTX1fyE7Nb6FYoURo/SPe62QCaAyzJvUjwRaIsc+NePBEniHlvxFmmX56+HZphIGtV0XeCirBtpDrTyQ==" }, + "quick-format-unescaped": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/quick-format-unescaped/-/quick-format-unescaped-4.0.1.tgz", + "integrity": "sha512-RyYpQ6Q5/drsJyOhrWHYMWTedvjTIat+FTwv0K4yoUxzvekw2aRHMQJLlnvt8UantkZg2++bEzD9EdxXqkWf4A==" + }, "range-parser": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", @@ -2767,6 +3028,15 @@ } } }, + "sonic-boom": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-1.3.0.tgz", + "integrity": "sha512-4nX6OYvOYr6R76xfQKi6cZpTO3YSWe/vd+QdIfoH0lBy0MnPkeAbb2rRWgmgADkXUeCKPwO1FZAKlAVWAadELw==", + "requires": { + "atomic-sleep": "^1.0.0", + "flatstr": "^1.0.12" + } + }, "source-map": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", @@ -2817,6 +3087,15 @@ "through": "2" } }, + "split2": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/split2/-/split2-3.2.2.tgz", + "integrity": "sha512-9NThjpgZnifTkJpzTZ7Eue85S49QwpNhZTq6GRJwObb6jnLFNGB7Qm73V5HewTROPyxD0C29xqmaI68bQtV+hg==", + "dev": true, + "requires": { + "readable-stream": "^3.0.0" + } + }, "sprintf-js": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", diff --git a/server/package.json b/server/package.json index dd8e184..9b03a26 100644 --- a/server/package.json +++ b/server/package.json @@ -9,18 +9,29 @@ "lint": "eslint . --ext .ts", "fix": "eslint . --ext .ts --fix" }, + "nodemonConfig": { + "ignore": [ + "utils/pptr_userDataDir", + "utils/output.json", + "utils/output_test.json" + ] + }, "dependencies": { "@types/body-parser": "^1.19.0", "@types/dotenv": "^8.2.0", "@types/express": "^4.17.8", + "@types/express-pino-logger": "^4.0.2", "@types/node": "^14.11.8", "@types/pg": "^7.14.5", + "@types/pino": "^6.3.3", "@types/puppeteer": "^3.0.2", "body-parser": "^1.19.0", "dotenv": "^8.2.0", "express": "^4.17.1", + "express-pino-logger": "^5.0.0", "nodemon": "^2.0.4", "pg": "^8.4.1", + "pino": "^6.7.0", "puppeteer": "^5.3.1", "ts-node": "^9.0.0", "ts-node-dev": "^1.0.0-pre.63", @@ -31,6 +42,7 @@ "@typescript-eslint/eslint-plugin-tslint": "^4.5.0", "@typescript-eslint/parser": "^4.5.0", "eslint": "^7.11.0", + "pino-pretty": "^4.3.0", "tslint": "^6.1.3", "tslint-microsoft-contrib": "^6.2.0" } diff --git a/server/src/database/db.ts b/server/src/database/db.ts index 6ed5323..0f72482 100644 --- a/server/src/database/db.ts +++ b/server/src/database/db.ts @@ -1,7 +1,10 @@ import { Pool, QueryResult } from "pg"; import config from "./config"; +import parentLogger from "../../utils/logger"; import localconfig from "./localconfig"; +const log = parentLogger.child({ module: "postgres" }); + class Database { private pool: Pool; @@ -9,7 +12,7 @@ class Database { // If on docker pool should take config, if on local database pool should take localconfig this.pool = new Pool(config); this.pool.on("error", (err, client) => { - console.error("Unexpected error on idle PostgreSQL client.", err); + log.error("Unexpected error on idle PostgreSQL client.", err); process.exit(-1); }); } @@ -21,7 +24,7 @@ class Database { client.release(); return queryRes; } catch (e) { - console.log("async query error"); + log.error("async query error"); client.release(); throw (e); } diff --git a/server/src/index.ts b/server/src/index.ts index 5ae1283..15d96f0 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -1,26 +1,30 @@ import express from "express"; import bodyParser from "body-parser"; import db from "./database/db"; +import expressPino from "express-pino-logger"; +import parentLogger from "../utils/logger"; import scraper from "../utils/scraper"; +const log = parentLogger.child({ module: "router" }); +const expressLogger = expressPino(log); const PORT = process.env.PORT || 5000; const app = express(); app.use(bodyParser.json()); +app.use(expressLogger); app.listen(PORT, () => { - console.log(`Server running on port ${PORT}`); + log.info(`Server running on port ${PORT}`); }); async function testDb() { try { const { rows } = await db.query("SELECT NOW()"); - console.log(`Postgresql connected ${rows[0].now}`); + log.info(`Postgresql connected ${rows[0].now}`); } catch (e) { - console.log(`error ${e}`); + log.error(`error ${e}`); } } // If docker isn't set up yet, this should error if you dont have postgres installed testDb(); - diff --git a/server/utils/logger.ts b/server/utils/logger.ts new file mode 100644 index 0000000..66dcea5 --- /dev/null +++ b/server/utils/logger.ts @@ -0,0 +1,22 @@ +import pino from "pino"; + +let environment = (process.env.NODE_ENV) || "development"; + +// Parent logger. If you want to make use of logging in a new file, you MUST import this +// and create a child logger with constructor argument {module: "modulename"} +const logger = pino({ + /** + * Sets the log level as LOG_LEVEL in env or defaults to "info" + * If set to "info", all logs EXCEPT for "trace" and "debug" are shown + * Change LOG_LEVEL to "trace" if you would like to see all log levels, + * or "debug" if you want to see everything but trace logs. + */ + level: process.env.LOG_LEVEL || "info", + /** + * Makes pretty (non-JSON) logs output to stdout. Should be disabled in + * production + */ + prettyPrint: (environment === "development") +}); + +export default logger; diff --git a/server/utils/scraper.ts b/server/utils/scraper.ts index 01d3a5c..7f770c7 100644 --- a/server/utils/scraper.ts +++ b/server/utils/scraper.ts @@ -1,6 +1,7 @@ import { Browser, ElementHandle, JSHandle, LaunchOptions, Page } from "puppeteer"; // types for TS import puppeteer from "puppeteer"; import fs from "fs"; +import parentLogger from "./logger"; interface TermTime { term: string; @@ -25,8 +26,14 @@ interface Course { sections: Section[]; } +const log = parentLogger.child({ module: "scraper" }); const courseCodeRegex = /([A-Z][A-Z][A-Z][A-Z]?\s\d\d\d[A-Z]?)/g; +let parseHrtimeToSeconds = (hrtime: number[]) => { + let seconds = (hrtime[0] + (hrtime[1] / 1e9)).toFixed(3); + return seconds; +}; + // helper function let getInnerHTML = async (element: ElementHandle | Page, selector: string): Promise => { return await element.$eval(selector, (elm: any) => elm.innerHTML.trim()); @@ -82,12 +89,16 @@ let getCourseUrls = async (url: string, browser: Browser): Promise => * @returns {Promise} a promise for a Course object */ let getCourseInfo = async (url: string, browser: Browser): Promise => { - console.time("[getCourseInfo] Opening course page"); + // to benchmark page opening times, since it's the biggest bottleneck for scraper currently + const startTime = process.hrtime(); let coursePage: Page = await browser.newPage(); await pageConfig(coursePage); await coursePage.goto(url); - console.timeEnd("[getCourseInfo] Opening course page"); + + const elapsedSeconds = parseHrtimeToSeconds(process.hrtime(startTime)); + log.info(`Opening course page: ${elapsedSeconds}s`); + let courseCode: string = await getInnerHTML(coursePage, ".breadcrumb.expand > li:nth-child(4)"); let courseTitle: string = await getInnerHTML(coursePage, ".content.expand > h4"); @@ -294,6 +305,9 @@ let scraper = async (subjectTest?: number) => { let puppeteerOptions: LaunchOptions = { // args should help reduce load on CPU, since we're running headless Chromium args: [ + "--no-sandbox", // disabled for docker container + "--disable-setuid-sandbox", // a sandbox dependent + "--no-zygote", // sandbox dependent "--disable-accelerated-2d-canvas", // disables gpu-accel 2d "--no-first-run", // disables first run options "--disable-gpu" // yes @@ -303,8 +317,8 @@ let scraper = async (subjectTest?: number) => { }; try { - console.time("scraper"); - console.log("scraper: Started scraping."); + let startTime = process.hrtime(); + log.info("Started scraping."); const browser: Browser = await puppeteer.launch(puppeteerOptions); const page = await browser.newPage(); await pageConfig(page); @@ -316,44 +330,47 @@ let scraper = async (subjectTest?: number) => { if (subjectTest !== undefined && subjectTest < 237 && subjectTest >= 0) { let courseUrls: string[] = await getCourseUrls(subjectUrls[subjectTest], browser); for (let i = 0; i < courseUrls.length; i++) { - console.time("[getCourseInfo] Build course information"); + const courseStartTime = process.hrtime(); let courseInfo = await getCourseInfo(courseUrls[i], browser); data.push(courseInfo); - console.timeEnd("[getCourseInfo] Build course information"); - console.log(`Pushed ${courseInfo.courseCode} (${i} of ${courseUrls.length})`); + const elapsedSeconds = parseHrtimeToSeconds(process.hrtime(courseStartTime)); + log.info(`getCourseInfo: build ${elapsedSeconds}s`); + log.info(`Pushed ${courseInfo.courseCode} (${i} of ${courseUrls.length})`); } } else if (subjectTest === undefined) { for (let i = 0; i < subjectUrls.length; i++) { let courseUrls: string[] = await getCourseUrls(subjectUrls[i], browser); for (let j = 0; j < courseUrls.length; j++) { - console.time("[getCourseInfo] Build course information"); + const courseStartTime = process.hrtime(); let courseInfo = await getCourseInfo(courseUrls[j], browser); data.push(courseInfo); - console.timeEnd("[getCourseInfo] Build course information"); - console.log(`Pushed ${courseInfo.courseCode} (${j + 1} of ${courseUrls.length} sections)`); + const elapsedSeconds = parseHrtimeToSeconds(process.hrtime(courseStartTime)); + log.info(`getCourseInfo: build ${elapsedSeconds}s`); + log.info(`Pushed ${courseInfo.courseCode} (${j + 1} of ${courseUrls.length} sections)`); } - console.log(`Pushed ${data[i].courseCode.substr(0, 4)} (${i + 1} of ${subjectUrls.length} courses)`); + log.info(`Pushed ${data[i].courseCode.substr(0, 4)} (${i + 1} of ${subjectUrls.length} courses)`); } } else { - console.error("scraper: Input must be a number in the range [0, 237] inclusive."); + log.error("Input must be a number in the range [0, 237] inclusive."); } if (data) { fs.writeFile(outputPath, JSON.stringify(data), (err: any) => { - if (err) return console.error(err); - console.log(`scraper: Finished scraping, wrote to ${outputPath}.`); + if (err) return log.error(err); + log.info(`Finished scraping, wrote to ${outputPath}.`); }); // update db // axios.post("webhook url", data); <-- post request to trigger front-end build script } else { - console.log("scraper: Possible error, nothing written to disk."); + log.warn("Possible error, nothing written to disk."); } - console.timeEnd("scraper"); // latest version: 13684777.998ms or uh... 3.8 hours lol + let scraperElapsedSeconds = parseHrtimeToSeconds(process.hrtime(startTime)); + log.info(`scraper: elapsed time ${scraperElapsedSeconds}s`); await browser.close(); } catch (err) { - console.error(err); + log.error(err); } };