From b1e686c2d89b60b65f2a51dd57494591573e32be Mon Sep 17 00:00:00 2001 From: Rexios Date: Mon, 20 Nov 2023 13:13:33 -0500 Subject: [PATCH] Move `pub_stats_collector` and `proxy` to this project --- proxy/.dockerignore | 9 + proxy/.gcloudignore | 4 + proxy/.gitignore | 9 + proxy/Dockerfile | 24 + proxy/README.md | 12 + proxy/analysis_options.yaml | 1 + proxy/bin/server.dart | 28 + proxy/deploy.sh | 9 + proxy/pubspec.lock | 437 +++++++++ proxy/pubspec.yaml | 18 + pub_stats_collector/.dockerignore | 9 + pub_stats_collector/.gcloudignore | 4 + pub_stats_collector/.gitignore | 18 + pub_stats_collector/Dockerfile | 23 + pub_stats_collector/README.md | 5 + pub_stats_collector/analysis_options.yaml | 1 + .../bin/pub_stats_collector.dart | 18 + pub_stats_collector/build.yaml | 7 + .../controller/score_fetch_controller.dart | 135 +++ .../lib/credential/credentials.dart | 29 + pub_stats_collector/lib/functions.dart | 56 ++ pub_stats_collector/lib/model/diff.dart | 8 + .../lib/repo/database_repo.dart | 66 ++ .../lib/repo/discord_repo.dart | 129 +++ pub_stats_collector/lib/repo/pub_repo.dart | 79 ++ .../lib/service/firebase_service.dart | 29 + pub_stats_collector/pubspec.lock | 878 ++++++++++++++++++ pub_stats_collector/pubspec.yaml | 32 + pub_stats_collector/test/fetch_test.dart | 9 + pub_stats_collector/tool/prune.dart | 67 ++ 30 files changed, 2153 insertions(+) create mode 100644 proxy/.dockerignore create mode 100644 proxy/.gcloudignore create mode 100644 proxy/.gitignore create mode 100644 proxy/Dockerfile create mode 100644 proxy/README.md create mode 100644 proxy/analysis_options.yaml create mode 100644 proxy/bin/server.dart create mode 100755 proxy/deploy.sh create mode 100644 proxy/pubspec.lock create mode 100644 proxy/pubspec.yaml create mode 100644 pub_stats_collector/.dockerignore create mode 100644 pub_stats_collector/.gcloudignore create mode 100644 pub_stats_collector/.gitignore create mode 100644 pub_stats_collector/Dockerfile create mode 100644 pub_stats_collector/README.md create mode 100644 pub_stats_collector/analysis_options.yaml create mode 100644 pub_stats_collector/bin/pub_stats_collector.dart create mode 100644 pub_stats_collector/build.yaml create mode 100644 pub_stats_collector/lib/controller/score_fetch_controller.dart create mode 100644 pub_stats_collector/lib/credential/credentials.dart create mode 100644 pub_stats_collector/lib/functions.dart create mode 100644 pub_stats_collector/lib/model/diff.dart create mode 100644 pub_stats_collector/lib/repo/database_repo.dart create mode 100644 pub_stats_collector/lib/repo/discord_repo.dart create mode 100644 pub_stats_collector/lib/repo/pub_repo.dart create mode 100644 pub_stats_collector/lib/service/firebase_service.dart create mode 100644 pub_stats_collector/pubspec.lock create mode 100644 pub_stats_collector/pubspec.yaml create mode 100644 pub_stats_collector/test/fetch_test.dart create mode 100644 pub_stats_collector/tool/prune.dart diff --git a/proxy/.dockerignore b/proxy/.dockerignore new file mode 100644 index 0000000..0c737e8 --- /dev/null +++ b/proxy/.dockerignore @@ -0,0 +1,9 @@ +.dart_tool/ +.dockerignore +.git/ +.github/ +.gitignore +.idea/ +.packages +Dockerfile +build/ diff --git a/proxy/.gcloudignore b/proxy/.gcloudignore new file mode 100644 index 0000000..2b05215 --- /dev/null +++ b/proxy/.gcloudignore @@ -0,0 +1,4 @@ +.gcloudignore +.git +.gitignore +#!include:.gitignore diff --git a/proxy/.gitignore b/proxy/.gitignore new file mode 100644 index 0000000..3947667 --- /dev/null +++ b/proxy/.gitignore @@ -0,0 +1,9 @@ +# Files and directories created by pub. +.dart_tool/ +.packages + +# Conventional directory for build output. +build/ + +# macOS +.DS_Store diff --git a/proxy/Dockerfile b/proxy/Dockerfile new file mode 100644 index 0000000..7124b49 --- /dev/null +++ b/proxy/Dockerfile @@ -0,0 +1,24 @@ +# Official Dart image: https://hub.docker.com/_/dart +# Specify the Dart SDK base image version using dart: (ex: dart:2.14) +FROM dart:stable AS build + +# Resolve app dependencies. +WORKDIR /app +COPY pubspec.* ./ +RUN dart pub get + +# Copy app source code and AOT compile it. +COPY . . +# Ensure packages are still up-to-date if anything has changed +RUN dart pub get --offline +RUN dart compile exe bin/server.dart -o bin/server + +# Build minimal serving image from AOT-compiled `/server` and required system +# libraries and configuration files stored in `/runtime/` from the build stage. +FROM scratch +COPY --from=build /runtime/ / +COPY --from=build /app/bin/server /app/bin/ + +# Start server. +EXPOSE 8080 +ENTRYPOINT ["/app/bin/server"] diff --git a/proxy/README.md b/proxy/README.md new file mode 100644 index 0000000..e6be239 --- /dev/null +++ b/proxy/README.md @@ -0,0 +1,12 @@ +Template for creating Google Cloud Run applications in Dart. Based on documentation from https://github.com/GoogleCloudPlatform/functions-framework-dart. + +The functions framework is nice for very simple applications, but isn't great if you want more control over your server behavior. + +## Prerequisites +- Have a general idea how to set up Cloud Run. This is not comprehensive documentation. + +## Usage +1. Create a new project based on this template +2. Compose your server in `bin/server.dart` +3. Edit `deploy.sh` to use the desired parameters +4. Run `deploy.sh` to deploy your application \ No newline at end of file diff --git a/proxy/analysis_options.yaml b/proxy/analysis_options.yaml new file mode 100644 index 0000000..121518d --- /dev/null +++ b/proxy/analysis_options.yaml @@ -0,0 +1 @@ +include: package:rexios_lints/dart/core.yaml \ No newline at end of file diff --git a/proxy/bin/server.dart b/proxy/bin/server.dart new file mode 100644 index 0000000..06f22d4 --- /dev/null +++ b/proxy/bin/server.dart @@ -0,0 +1,28 @@ +import 'dart:io'; +import 'package:shelf/shelf.dart'; +import 'package:shelf/shelf_io.dart' as shelf_io; +import 'package:shelf_cors_headers/shelf_cors_headers.dart'; +import 'package:shelf_proxy/shelf_proxy.dart'; +import 'package:shelf_router/shelf_router.dart'; + +final pubHandler = Pipeline() + .addMiddleware( + corsHeaders( + originChecker: + originOneOf(['https://pubstats.dev', 'https://beta.pubstats.dev']), + ), + ) + .addHandler( + proxyHandler('https://pub.dartlang.org', proxyName: 'proxy.pubstats.dev'), + ); + +void main(List arguments) async { + final app = Router()..mount('/pub', pubHandler); + + final port = Platform.environment.containsKey('PORT') + ? int.parse(Platform.environment['PORT']!) + : 8080; + final server = await shelf_io.serve(app.call, '0.0.0.0', port); + + print('Listening on :${server.port}'); +} diff --git a/proxy/deploy.sh b/proxy/deploy.sh new file mode 100755 index 0000000..6ae0569 --- /dev/null +++ b/proxy/deploy.sh @@ -0,0 +1,9 @@ +# gcloud run deploy NAME \ # the Cloud Run service name +# --source=PATH \ # can use $PWD or . for current dir +# --project=PROJECT \ # the Google Cloud project ID +# --region=REGION \ # ex: us-central1 +# --platform=managed \ # for Cloud Run +# --allow-unauthenticated # for public access + +# ex: gcloud run deploy my-service --source=. --project=my-project --region=us-central1 --platform=managed --allow-unauthenticated +gcloud run deploy pub-stats-collector-nginx --source=. --project=pub-stats-collector --region=us-central1 --platform=managed --allow-unauthenticated \ No newline at end of file diff --git a/proxy/pubspec.lock b/proxy/pubspec.lock new file mode 100644 index 0000000..7a0af7c --- /dev/null +++ b/proxy/pubspec.lock @@ -0,0 +1,437 @@ +# Generated by pub +# See https://dart.dev/tools/pub/glossary#lockfile +packages: + _fe_analyzer_shared: + dependency: transitive + description: + name: _fe_analyzer_shared + sha256: "36a321c3d2cbe01cbcb3540a87b8843846e0206df3e691fa7b23e19e78de6d49" + url: "https://pub.dev" + source: hosted + version: "65.0.0" + analyzer: + dependency: transitive + description: + name: analyzer + sha256: dfe03b90ec022450e22513b5e5ca1f01c0c01de9c3fba2f7fd233cb57a6b9a07 + url: "https://pub.dev" + source: hosted + version: "6.3.0" + args: + dependency: transitive + description: + name: args + sha256: eef6c46b622e0494a36c5a12d10d77fb4e855501a91c1b9ef9339326e58f0596 + url: "https://pub.dev" + source: hosted + version: "2.4.2" + async: + dependency: transitive + description: + name: async + sha256: "947bfcf187f74dbc5e146c9eb9c0f10c9f8b30743e341481c1e2ed3ecc18c20c" + url: "https://pub.dev" + source: hosted + version: "2.11.0" + boolean_selector: + dependency: transitive + description: + name: boolean_selector + sha256: "6cfb5af12253eaf2b368f07bacc5a80d1301a071c73360d746b7f2e32d762c66" + url: "https://pub.dev" + source: hosted + version: "2.1.1" + collection: + dependency: transitive + description: + name: collection + sha256: ee67cb0715911d28db6bf4af1026078bd6f0128b07a5f66fb2ed94ec6783c09a + url: "https://pub.dev" + source: hosted + version: "1.18.0" + convert: + dependency: transitive + description: + name: convert + sha256: "0f08b14755d163f6e2134cb58222dd25ea2a2ee8a195e53983d57c075324d592" + url: "https://pub.dev" + source: hosted + version: "3.1.1" + coverage: + dependency: transitive + description: + name: coverage + sha256: ac86d3abab0f165e4b8f561280ff4e066bceaac83c424dd19f1ae2c2fcd12ca9 + url: "https://pub.dev" + source: hosted + version: "1.7.1" + crypto: + dependency: transitive + description: + name: crypto + sha256: ff625774173754681d66daaf4a448684fb04b78f902da9cb3d308c19cc5e8bab + url: "https://pub.dev" + source: hosted + version: "3.0.3" + file: + dependency: transitive + description: + name: file + sha256: "5fc22d7c25582e38ad9a8515372cd9a93834027aacf1801cf01164dac0ffa08c" + url: "https://pub.dev" + source: hosted + version: "7.0.0" + flutter_lints: + dependency: transitive + description: + name: flutter_lints + sha256: e2a421b7e59244faef694ba7b30562e489c2b489866e505074eb005cd7060db7 + url: "https://pub.dev" + source: hosted + version: "3.0.1" + frontend_server_client: + dependency: transitive + description: + name: frontend_server_client + sha256: "408e3ca148b31c20282ad6f37ebfa6f4bdc8fede5b74bc2f08d9d92b55db3612" + url: "https://pub.dev" + source: hosted + version: "3.2.0" + glob: + dependency: transitive + description: + name: glob + sha256: "0e7014b3b7d4dac1ca4d6114f82bf1782ee86745b9b42a92c9289c23d8a0ab63" + url: "https://pub.dev" + source: hosted + version: "2.1.2" + http: + dependency: "direct dev" + description: + name: http + sha256: "759d1a329847dd0f39226c688d3e06a6b8679668e350e2891a6474f8b4bb8525" + url: "https://pub.dev" + source: hosted + version: "1.1.0" + http_methods: + dependency: transitive + description: + name: http_methods + sha256: "6bccce8f1ec7b5d701e7921dca35e202d425b57e317ba1a37f2638590e29e566" + url: "https://pub.dev" + source: hosted + version: "1.1.1" + http_multi_server: + dependency: transitive + description: + name: http_multi_server + sha256: "97486f20f9c2f7be8f514851703d0119c3596d14ea63227af6f7a481ef2b2f8b" + url: "https://pub.dev" + source: hosted + version: "3.2.1" + http_parser: + dependency: transitive + description: + name: http_parser + sha256: "2aa08ce0341cc9b354a498388e30986515406668dbcc4f7c950c3e715496693b" + url: "https://pub.dev" + source: hosted + version: "4.0.2" + io: + dependency: transitive + description: + name: io + sha256: "2ec25704aba361659e10e3e5f5d672068d332fc8ac516421d483a11e5cbd061e" + url: "https://pub.dev" + source: hosted + version: "1.0.4" + js: + dependency: transitive + description: + name: js + sha256: f2c445dce49627136094980615a031419f7f3eb393237e4ecd97ac15dea343f3 + url: "https://pub.dev" + source: hosted + version: "0.6.7" + lints: + dependency: transitive + description: + name: lints + sha256: cbf8d4b858bb0134ef3ef87841abdf8d63bfc255c266b7bf6b39daa1085c4290 + url: "https://pub.dev" + source: hosted + version: "3.0.0" + logging: + dependency: transitive + description: + name: logging + sha256: "623a88c9594aa774443aa3eb2d41807a48486b5613e67599fb4c41c0ad47c340" + url: "https://pub.dev" + source: hosted + version: "1.2.0" + matcher: + dependency: transitive + description: + name: matcher + sha256: "1803e76e6653768d64ed8ff2e1e67bea3ad4b923eb5c56a295c3e634bad5960e" + url: "https://pub.dev" + source: hosted + version: "0.12.16" + meta: + dependency: transitive + description: + name: meta + sha256: d584fa6707a52763a52446f02cc621b077888fb63b93bbcb1143a7be5a0c0c04 + url: "https://pub.dev" + source: hosted + version: "1.11.0" + mime: + dependency: transitive + description: + name: mime + sha256: e4ff8e8564c03f255408decd16e7899da1733852a9110a58fe6d1b817684a63e + url: "https://pub.dev" + source: hosted + version: "1.0.4" + node_preamble: + dependency: transitive + description: + name: node_preamble + sha256: "6e7eac89047ab8a8d26cf16127b5ed26de65209847630400f9aefd7cd5c730db" + url: "https://pub.dev" + source: hosted + version: "2.0.2" + package_config: + dependency: transitive + description: + name: package_config + sha256: "1c5b77ccc91e4823a5af61ee74e6b972db1ef98c2ff5a18d3161c982a55448bd" + url: "https://pub.dev" + source: hosted + version: "2.1.0" + path: + dependency: transitive + description: + name: path + sha256: "8829d8a55c13fc0e37127c29fedf290c102f4e40ae94ada574091fe0ff96c917" + url: "https://pub.dev" + source: hosted + version: "1.8.3" + pool: + dependency: transitive + description: + name: pool + sha256: "20fe868b6314b322ea036ba325e6fc0711a22948856475e2c2b6306e8ab39c2a" + url: "https://pub.dev" + source: hosted + version: "1.5.1" + pub_semver: + dependency: transitive + description: + name: pub_semver + sha256: "40d3ab1bbd474c4c2328c91e3a7df8c6dd629b79ece4c4bd04bee496a224fb0c" + url: "https://pub.dev" + source: hosted + version: "2.1.4" + rexios_lints: + dependency: "direct dev" + description: + name: rexios_lints + sha256: "0e099c3ab11dca537ceef7668c2f9c50650c0551cf59858db4559ddaedcfbfd9" + url: "https://pub.dev" + source: hosted + version: "6.0.1" + shelf: + dependency: "direct main" + description: + name: shelf + sha256: ad29c505aee705f41a4d8963641f91ac4cee3c8fad5947e033390a7bd8180fa4 + url: "https://pub.dev" + source: hosted + version: "1.4.1" + shelf_cors_headers: + dependency: "direct main" + description: + name: shelf_cors_headers + sha256: a127c80f99bbef3474293db67a7608e3a0f1f0fcdb171dad77fa9bd2cd123ae4 + url: "https://pub.dev" + source: hosted + version: "0.1.5" + shelf_packages_handler: + dependency: transitive + description: + name: shelf_packages_handler + sha256: "89f967eca29607c933ba9571d838be31d67f53f6e4ee15147d5dc2934fee1b1e" + url: "https://pub.dev" + source: hosted + version: "3.0.2" + shelf_proxy: + dependency: "direct main" + description: + name: shelf_proxy + sha256: a71d2307f4393211930c590c3d2c00630f6c5a7a77edc1ef6436dfd85a6a7ee3 + url: "https://pub.dev" + source: hosted + version: "1.0.4" + shelf_router: + dependency: "direct main" + description: + name: shelf_router + sha256: f5e5d492440a7fb165fe1e2e1a623f31f734d3370900070b2b1e0d0428d59864 + url: "https://pub.dev" + source: hosted + version: "1.1.4" + shelf_static: + dependency: transitive + description: + name: shelf_static + sha256: a41d3f53c4adf0f57480578c1d61d90342cd617de7fc8077b1304643c2d85c1e + url: "https://pub.dev" + source: hosted + version: "1.1.2" + shelf_web_socket: + dependency: transitive + description: + name: shelf_web_socket + sha256: "9ca081be41c60190ebcb4766b2486a7d50261db7bd0f5d9615f2d653637a84c1" + url: "https://pub.dev" + source: hosted + version: "1.0.4" + source_map_stack_trace: + dependency: transitive + description: + name: source_map_stack_trace + sha256: "84cf769ad83aa6bb61e0aa5a18e53aea683395f196a6f39c4c881fb90ed4f7ae" + url: "https://pub.dev" + source: hosted + version: "2.1.1" + source_maps: + dependency: transitive + description: + name: source_maps + sha256: "708b3f6b97248e5781f493b765c3337db11c5d2c81c3094f10904bfa8004c703" + url: "https://pub.dev" + source: hosted + version: "0.10.12" + source_span: + dependency: transitive + description: + name: source_span + sha256: "53e943d4206a5e30df338fd4c6e7a077e02254531b138a15aec3bd143c1a8b3c" + url: "https://pub.dev" + source: hosted + version: "1.10.0" + stack_trace: + dependency: transitive + description: + name: stack_trace + sha256: "73713990125a6d93122541237550ee3352a2d84baad52d375a4cad2eb9b7ce0b" + url: "https://pub.dev" + source: hosted + version: "1.11.1" + stream_channel: + dependency: transitive + description: + name: stream_channel + sha256: ba2aa5d8cc609d96bbb2899c28934f9e1af5cddbd60a827822ea467161eb54e7 + url: "https://pub.dev" + source: hosted + version: "2.1.2" + string_scanner: + dependency: transitive + description: + name: string_scanner + sha256: "556692adab6cfa87322a115640c11f13cb77b3f076ddcc5d6ae3c20242bedcde" + url: "https://pub.dev" + source: hosted + version: "1.2.0" + term_glyph: + dependency: transitive + description: + name: term_glyph + sha256: a29248a84fbb7c79282b40b8c72a1209db169a2e0542bce341da992fe1bc7e84 + url: "https://pub.dev" + source: hosted + version: "1.2.1" + test: + dependency: "direct dev" + description: + name: test + sha256: a1f7595805820fcc05e5c52e3a231aedd0b72972cb333e8c738a8b1239448b6f + url: "https://pub.dev" + source: hosted + version: "1.24.9" + test_api: + dependency: transitive + description: + name: test_api + sha256: "5c2f730018264d276c20e4f1503fd1308dfbbae39ec8ee63c5236311ac06954b" + url: "https://pub.dev" + source: hosted + version: "0.6.1" + test_core: + dependency: transitive + description: + name: test_core + sha256: a757b14fc47507060a162cc2530d9a4a2f92f5100a952c7443b5cad5ef5b106a + url: "https://pub.dev" + source: hosted + version: "0.5.9" + test_process: + dependency: "direct dev" + description: + name: test_process + sha256: "217f19b538926e4922bdb2a01410100ec4e3beb4cc48eae5ae6b20037b07bbd6" + url: "https://pub.dev" + source: hosted + version: "2.1.0" + typed_data: + dependency: transitive + description: + name: typed_data + sha256: facc8d6582f16042dd49f2463ff1bd6e2c9ef9f3d5da3d9b087e244a7b564b3c + url: "https://pub.dev" + source: hosted + version: "1.3.2" + vm_service: + dependency: transitive + description: + name: vm_service + sha256: b3d56ff4341b8f182b96aceb2fa20e3dcb336b9f867bc0eafc0de10f1048e957 + url: "https://pub.dev" + source: hosted + version: "13.0.0" + watcher: + dependency: transitive + description: + name: watcher + sha256: "3d2ad6751b3c16cf07c7fca317a1413b3f26530319181b37e3b9039b84fc01d8" + url: "https://pub.dev" + source: hosted + version: "1.1.0" + web_socket_channel: + dependency: transitive + description: + name: web_socket_channel + sha256: d88238e5eac9a42bb43ca4e721edba3c08c6354d4a53063afaa568516217621b + url: "https://pub.dev" + source: hosted + version: "2.4.0" + webkit_inspection_protocol: + dependency: transitive + description: + name: webkit_inspection_protocol + sha256: "87d3f2333bb240704cd3f1c6b5b7acd8a10e7f0bc28c28dcf14e782014f4a572" + url: "https://pub.dev" + source: hosted + version: "1.2.1" + yaml: + dependency: transitive + description: + name: yaml + sha256: "75769501ea3489fca56601ff33454fe45507ea3bfb014161abc3b43ae25989d5" + url: "https://pub.dev" + source: hosted + version: "3.1.2" +sdks: + dart: ">=3.0.0 <4.0.0" diff --git a/proxy/pubspec.yaml b/proxy/pubspec.yaml new file mode 100644 index 0000000..bcdcd76 --- /dev/null +++ b/proxy/pubspec.yaml @@ -0,0 +1,18 @@ +name: dart_cloud_run_template +version: 1.0.0 +publish_to: none + +environment: + sdk: ">=2.19.0 <3.0.0" + +dependencies: + shelf: ^1.3.2 + shelf_router: ^1.1.3 + shelf_proxy: ^1.0.2 + shelf_cors_headers: ^0.1.3 + +dev_dependencies: + rexios_lints: ^6.0.1 + test: ^1.15.7 + test_process: ^2.0.0 + http: ^1.1.0 diff --git a/pub_stats_collector/.dockerignore b/pub_stats_collector/.dockerignore new file mode 100644 index 0000000..0c737e8 --- /dev/null +++ b/pub_stats_collector/.dockerignore @@ -0,0 +1,9 @@ +.dart_tool/ +.dockerignore +.git/ +.github/ +.gitignore +.idea/ +.packages +Dockerfile +build/ diff --git a/pub_stats_collector/.gcloudignore b/pub_stats_collector/.gcloudignore new file mode 100644 index 0000000..1679441 --- /dev/null +++ b/pub_stats_collector/.gcloudignore @@ -0,0 +1,4 @@ +.gcloudignore +.git +.gitignore +#!include:.gitignore \ No newline at end of file diff --git a/pub_stats_collector/.gitignore b/pub_stats_collector/.gitignore new file mode 100644 index 0000000..e6da9bc --- /dev/null +++ b/pub_stats_collector/.gitignore @@ -0,0 +1,18 @@ +# Files and directories created by pub. +.dart_tool/ +.packages + +# Conventional directory for build output. +build/ + +# macOS +.DS_Store + +# Generated Cloud Run server +bin/server.dart + +# Pruned database file +pruned.json + +# Secrets +secret \ No newline at end of file diff --git a/pub_stats_collector/Dockerfile b/pub_stats_collector/Dockerfile new file mode 100644 index 0000000..88459c9 --- /dev/null +++ b/pub_stats_collector/Dockerfile @@ -0,0 +1,23 @@ +FROM dart:stable AS build + +# Resolve app dependencies. +WORKDIR /app +COPY pubspec.* ./ +RUN dart pub get + +# Copy app source code and AOT compile it. +COPY . . +# Ensure packages are still up-to-date if anything has changed +RUN dart pub get --offline +RUN dart pub run build_runner build --delete-conflicting-outputs +RUN dart compile exe bin/pub_stats_collector.dart -o bin/server + +# Build minimal serving image from AOT-compiled `/server` and required system +# libraries and configuration files stored in `/runtime/` from the build stage. +FROM scratch +COPY --from=build /runtime/ / +COPY --from=build /app/bin/server /app/bin/ + +# Start server. +EXPOSE 8080 +CMD ["/app/bin/server"] diff --git a/pub_stats_collector/README.md b/pub_stats_collector/README.md new file mode 100644 index 0000000..2b6e925 --- /dev/null +++ b/pub_stats_collector/README.md @@ -0,0 +1,5 @@ +### pub-stats-collector +```console +gcloud config set project pub-stats-collector +gcloud beta run deploy pub-stats-collector --source=. +``` diff --git a/pub_stats_collector/analysis_options.yaml b/pub_stats_collector/analysis_options.yaml new file mode 100644 index 0000000..121518d --- /dev/null +++ b/pub_stats_collector/analysis_options.yaml @@ -0,0 +1 @@ +include: package:rexios_lints/dart/core.yaml \ No newline at end of file diff --git a/pub_stats_collector/bin/pub_stats_collector.dart b/pub_stats_collector/bin/pub_stats_collector.dart new file mode 100644 index 0000000..c4e69eb --- /dev/null +++ b/pub_stats_collector/bin/pub_stats_collector.dart @@ -0,0 +1,18 @@ +import 'package:functions_framework/serve.dart'; +import 'package:pub_stats_collector/functions.dart' as functions; + +/// Boiler plate code for the functions framework +/// +/// You could probably do some fancy things here, but this is just a simple example +Future main(List args) async { + await serve(args, _nameToFunctionTarget); +} + +FunctionTarget? _nameToFunctionTarget(String name) { + switch (name) { + case 'function': + return FunctionTarget.http(functions.function); + default: + return null; + } +} diff --git a/pub_stats_collector/build.yaml b/pub_stats_collector/build.yaml new file mode 100644 index 0000000..f206ed1 --- /dev/null +++ b/pub_stats_collector/build.yaml @@ -0,0 +1,7 @@ +targets: + $default: + builders: + source_gen|combining_builder: + options: + ignore_for_file: + - require_trailing_commas \ No newline at end of file diff --git a/pub_stats_collector/lib/controller/score_fetch_controller.dart b/pub_stats_collector/lib/controller/score_fetch_controller.dart new file mode 100644 index 0000000..4f9876b --- /dev/null +++ b/pub_stats_collector/lib/controller/score_fetch_controller.dart @@ -0,0 +1,135 @@ +import 'dart:async'; + +import 'package:pub_api_client/pub_api_client.dart' hide Credentials; +import 'package:pub_stats_collector/credential/credentials.dart'; +import 'package:pub_stats_collector/model/diff.dart'; +import 'package:pub_stats_collector/repo/database_repo.dart'; +import 'package:pub_stats_collector/repo/discord_repo.dart'; +import 'package:pub_stats_collector/repo/pub_repo.dart'; +import 'package:pub_stats_core/pub_stats_core.dart'; + +class ScoreFetchController { + final PubRepo _pub; + final DatabaseRepo _database; + final DiscordRepo _discord; + final Map> _alertConfigs; + + ScoreFetchController( + Credentials credentials, + this._database, + this._discord, + this._alertConfigs, + ) : _pub = PubRepo(credentials); + + Future fetchScores() async { + final startTime = DateTime.now(); + final globalStats = await _pub.fetchAllScores(_handleScore); + + await _database.writeGlobalStats(globalStats); + print('Global stats:'); + print(globalStats.toJson()); + + final duration = DateTime.now().difference(startTime); + print('Package processing completed in ${duration.inSeconds} seconds'); + + for (final config in _alertConfigs['.system'] ?? []) { + await switch (config.type) { + AlertConfigType.discord => _discord.sendGlobalStatsAlert( + config: config as DiscordAlertConfig, + stats: globalStats, + duration: duration, + ), + }; + } + print('Discord handling completed'); + } + + Future _handleScore(String package, PackageScore score) async { + final miniScore = MiniPackageScore.fromPackageScore(score); + // Don't write unprocessed scores + if (miniScore == null) return; + + final previousScore = await _database.readLatestPackageScore(package); + + // Send alerts + final publisher = await _pub.fetchPublisher(package); + final configs = [ + ..._alertConfigs[package] ?? [], + if (publisher != null) ..._alertConfigs['publisher:$publisher'] ?? [], + ]; + + await Future.wait( + configs.map( + (e) => processAlert( + package: package, + configs: configs, + miniScore: miniScore, + score: score, + previousScore: previousScore, + ), + ), + ); + + return _database.writePackageScore( + name: package, + lastUpdated: score.lastUpdated, + score: miniScore, + ); + } + + Future processAlert({ + required String package, + required List configs, + required MiniPackageScore miniScore, + required PackageScore score, + required MiniPackageScore? previousScore, + }) async { + final Map changes; + if (previousScore != null) { + changes = { + PackageDataField.likeCount: Diff( + previousScore.likeCount, + miniScore.likeCount, + ), + PackageDataField.popularityScore: Diff( + previousScore.popularityScore, + miniScore.popularityScore, + ), + }..removeWhere((key, value) => !value.isDifferent); + } else { + changes = {}; + } + + final warnings = {}; + final grantedPoints = score.grantedPoints; + final maxPoints = score.maxPoints; + if (grantedPoints != null && + maxPoints != null && + grantedPoints < maxPoints) { + warnings[PackageDataField.pubPoints] = + 'Only $grantedPoints/$maxPoints pub points'; + } + + if (changes.isEmpty && warnings.isEmpty) return; + + for (final config in configs) { + final filteredChanges = Map.fromEntries( + changes.entries.where((e) => !config.ignore.contains(e.key)), + ); + final filteredWarnings = Map.fromEntries( + warnings.entries.where((e) => !config.ignore.contains(e.key)), + ); + + if (filteredChanges.isEmpty && filteredWarnings.isEmpty) continue; + + await switch (config.type) { + AlertConfigType.discord => _discord.sendPackageAlert( + package: package, + config: config as DiscordAlertConfig, + changes: filteredChanges, + warnings: filteredWarnings.values.toList(), + ), + }; + } + } +} diff --git a/pub_stats_collector/lib/credential/credentials.dart b/pub_stats_collector/lib/credential/credentials.dart new file mode 100644 index 0000000..d4cd012 --- /dev/null +++ b/pub_stats_collector/lib/credential/credentials.dart @@ -0,0 +1,29 @@ +import 'dart:convert'; +import 'dart:io'; + +class Credentials { + static const _userAgent = 'Rexios80/pub_stats_collector'; + + static final prod = Credentials._( + userAgent: _userAgent, + serviceAccount: jsonDecode(Platform.environment['SERVICE_ACCOUNT_KEY']!), + databaseUrl: 'https://pub-stats-collector-default-rtdb.firebaseio.com/', + ); + + static final debug = Credentials._( + userAgent: _userAgent, + serviceAccount: + jsonDecode(File('secret/service_account_key.json').readAsStringSync()), + databaseUrl: 'http://127.0.0.1:9000/?ns=pub-stats-collector-default-rtdb', + ); + + final String userAgent; + final Map serviceAccount; + final String databaseUrl; + + const Credentials._({ + required this.userAgent, + required this.serviceAccount, + required this.databaseUrl, + }); +} diff --git a/pub_stats_collector/lib/functions.dart b/pub_stats_collector/lib/functions.dart new file mode 100644 index 0000000..e3ed63b --- /dev/null +++ b/pub_stats_collector/lib/functions.dart @@ -0,0 +1,56 @@ +import 'dart:async'; +import 'dart:io'; + +import 'package:functions_framework/functions_framework.dart'; +import 'package:pub_stats_collector/controller/score_fetch_controller.dart'; +import 'package:pub_stats_collector/credential/credentials.dart'; +import 'package:pub_stats_collector/repo/database_repo.dart'; +import 'package:pub_stats_collector/repo/discord_repo.dart'; +import 'package:pub_stats_collector/service/firebase_service.dart'; +import 'package:pub_stats_core/pub_stats_core.dart'; +import 'package:shelf/shelf.dart'; + +/// The entry point of the cloud function +@CloudFunction() +Future function(Request request, {bool debug = false}) async { + final credentials = debug ? Credentials.debug : Credentials.prod; + + final firebase = await FirebaseService.create(credentials); + final database = DatabaseRepo(firebase.database); + final discord = DiscordRepo(credentials); + + final rawConfigs = await database.readAlertConfigs(); + final alertConfigs = >{}; + for (final config in rawConfigs.values.expand((e) => e)) { + alertConfigs.update( + config.slug, + (value) => value..add(config), + ifAbsent: () => [config], + ); + } + + final controller = + ScoreFetchController(credentials, database, discord, alertConfigs); + + try { + await controller.fetchScores().timeout( + Duration(minutes: debug ? 10 : 5), + onTimeout: () => throw TimeoutException('Global timeout reached'), + ); + } catch (e, stacktrace) { + print(e); + print(stacktrace); + + for (final config in alertConfigs['.system'] ?? []) { + await switch (config.type) { + AlertConfigType.discord => discord.sendSystemErrorAlert( + config: config as DiscordAlertConfig, + error: e, + ) + }; + } + exit(1); + } + + return Response.ok(null); +} diff --git a/pub_stats_collector/lib/model/diff.dart b/pub_stats_collector/lib/model/diff.dart new file mode 100644 index 0000000..df83482 --- /dev/null +++ b/pub_stats_collector/lib/model/diff.dart @@ -0,0 +1,8 @@ +class Diff { + final Object before; + final Object after; + + Diff(this.before, this.after); + + bool get isDifferent => before != after; +} diff --git a/pub_stats_collector/lib/repo/database_repo.dart b/pub_stats_collector/lib/repo/database_repo.dart new file mode 100644 index 0000000..d34b2c0 --- /dev/null +++ b/pub_stats_collector/lib/repo/database_repo.dart @@ -0,0 +1,66 @@ +import 'package:firebase_admin/firebase_admin.dart'; +import 'package:pub_stats_core/pub_stats_core.dart'; + +class DatabaseRepo { + final Database _database; + + DatabaseRepo(this._database); + + Future readLatestPackageScore(String name) async { + final data = await _database + .ref() + .child('stats') + .child(name) + .orderByKey() + .limitToLast(1) + .once(); + final value = data.value as Map?; + if (value == null) return null; + return MiniPackageScore.fromJson(value.values.first); + } + + Future writePackageScore({ + required String name, + required DateTime lastUpdated, + required MiniPackageScore score, + }) { + return _database + .ref() + .child('stats') + .child(name) + // Store as seconds since epoch to save space + .child(lastUpdated.secondsSinceEpoch.toString()) + .set(score.toJson()); + } + + Future writeGlobalStats(GlobalStats stats) async { + await _database.ref().child('global_stats').set(stats.toJson()); + + await _database + .ref() + .child('package_counts') + .child(DateTime.now().secondsSinceEpoch.toString()) + .set(stats.packageCount); + } + + /// Map of `uid` to their list of [AlertConfig]s + Future>> readAlertConfigs() async { + final data = await _database.ref().child('alerts').once(); + final value = data.value as Map?; + if (value == null) return {}; + + return value.map( + (key, value) => MapEntry( + key, + (value as List) + .cast>() + .map(AlertConfig.fromJson) + .toList(), + ), + ); + } +} + +extension on DateTime { + int get secondsSinceEpoch => (millisecondsSinceEpoch / 1000).round(); +} diff --git a/pub_stats_collector/lib/repo/discord_repo.dart b/pub_stats_collector/lib/repo/discord_repo.dart new file mode 100644 index 0000000..aafcaa9 --- /dev/null +++ b/pub_stats_collector/lib/repo/discord_repo.dart @@ -0,0 +1,129 @@ +import 'package:discord_interactions/discord_interactions.dart'; +import 'package:pub_stats_collector/credential/credentials.dart'; +import 'package:pub_stats_collector/model/diff.dart'; +import 'package:pub_stats_core/pub_stats_core.dart'; +import 'package:recase/recase.dart'; + +class DiscordRepo { + static const _webhookUsername = 'Bartender'; + + final _discord = DiscordApi( + applicationId: '', + userAgent: DiscordUserAgent( + url: 'https://github.com/Rexios80/pub_stats_collector', + versionNumber: '1.0.0', + ), + botToken: '', + ); + + DiscordRepo(Credentials credentials); + + Future sendPackageAlert({ + required String package, + required DiscordAlertConfig config, + required Map changes, + required List warnings, + }) async { + final int color; + if (warnings.isNotEmpty) { + color = DiscordColor.red; + } else { + color = DiscordColor.purple; + } + + return _executeWebhook( + config.id, + token: config.token, + embeds: [ + Embed( + title: package, + color: color, + url: 'https://pub.dev/packages/$package', + fields: [ + ...changes.entries.map( + (e) => EmbedField( + name: e.key.name.titleCase, + value: '${e.value.before} -> ${e.value.after}', + ), + ), + if (warnings.isNotEmpty) + EmbedField( + name: 'Warnings', + value: warnings.join('\n'), + ), + ], + ), + ], + ); + } + + Future sendSystemErrorAlert({ + required DiscordAlertConfig config, + required Object error, + }) { + return _executeWebhook( + config.id, + token: config.token, + embeds: [ + Embed( + title: 'Error', + color: DiscordColor.red, + description: error.toString(), + ), + ], + ); + } + + Future sendGlobalStatsAlert({ + required DiscordAlertConfig config, + required GlobalStats stats, + required Duration duration, + }) { + return _executeWebhook( + config.id, + token: config.token, + embeds: [ + Embed( + title: 'Package Scan Completed', + color: DiscordColor.dartBlue, + url: 'https://pubstats.dev', + fields: [ + EmbedField( + name: 'Package Count', + value: stats.packageCount.toString(), + ), + EmbedField( + name: 'Most Liked Package', + value: stats.mostLikedPackage, + ), + EmbedField( + name: 'Most Popular Package', + value: stats.mostPopularPackage, + ), + EmbedField( + name: 'Execution Time', + value: '${duration.inSeconds} seconds', + ), + ], + footer: EmbedFooter( + text: 'See historical package stats on pubstats.dev', + iconUrl: 'https://pubstats.dev/splash/img/dark-2x.png', + ), + ), + ], + ); + } + + Future _executeWebhook( + String id, { + required String token, + required List embeds, + }) { + return _discord.webhooks.executeWebhook( + id, + token: token, + username: _webhookUsername, + embeds: embeds, + ); + } +} diff --git a/pub_stats_collector/lib/repo/pub_repo.dart b/pub_stats_collector/lib/repo/pub_repo.dart new file mode 100644 index 0000000..9534d84 --- /dev/null +++ b/pub_stats_collector/lib/repo/pub_repo.dart @@ -0,0 +1,79 @@ +import 'dart:async'; + +import 'package:flutter_tools_task_queue/flutter_tools_task_queue.dart'; +import 'package:pub_api_client/pub_api_client.dart' hide Credentials; +import 'package:pub_stats_collector/credential/credentials.dart'; +import 'package:pub_stats_core/pub_stats_core.dart'; + +typedef PubPackageScore = ({String name, PackageScore score}); + +class PubRepo { + final PubClient _client; + + PubRepo(Credentials credentials) + : _client = PubClient(client: UserAgentClient(credentials.userAgent)); + + Future fetchAllScores( + Future Function(String package, PackageScore score) handleScore, + ) async { + final packages = await _client.packageNames(); + print('Fetched ${packages.length} package names'); + + PubPackageScore? mostLikedPackage; + PubPackageScore? mostPopularPackage; + + var completed = 0; + Future fetchPackageData(String package) async { + final score = await _client.packageScore(package); + final packageScore = (name: package, score: score); + + if (score.likeCount > (mostLikedPackage?.score.likeCount ?? 0)) { + print('Most liked package: $package'); + mostLikedPackage = packageScore; + } + if ((score.popularityScore ?? 0) > + (mostPopularPackage?.score.popularityScore ?? 0)) { + print('Most popular package: $package'); + mostPopularPackage = packageScore; + } + + await handleScore(package, score).timeout( + Duration(seconds: 30), + onTimeout: () => + throw TimeoutException('Timeout handling score for $package'), + ); + + completed++; + if (completed % 100 == 0) { + print('Processed $completed/${packages.length} packages'); + } + } + + final queue = TaskQueue(maxJobs: 100); + for (final package in packages) { + unawaited( + queue.add(() async { + try { + await fetchPackageData(package); + } catch (e) { + print('Error processing package $package: $e'); + } + }), + ); + } + await queue.tasksComplete; + print('Processed all scores'); + + return GlobalStats( + packageCount: packages.length, + mostLikedPackage: mostLikedPackage?.name ?? '', + mostPopularPackage: mostPopularPackage?.name ?? '', + lastUpdated: DateTime.now().toUtc(), + ); + } + + Future fetchPublisher(String package) async { + final data = await _client.packagePublisher(package); + return data.publisherId; + } +} diff --git a/pub_stats_collector/lib/service/firebase_service.dart b/pub_stats_collector/lib/service/firebase_service.dart new file mode 100644 index 0000000..ee30f50 --- /dev/null +++ b/pub_stats_collector/lib/service/firebase_service.dart @@ -0,0 +1,29 @@ +import 'package:firebase_admin/firebase_admin.dart' hide Credentials; +import 'package:pub_stats_collector/credential/credentials.dart'; + +class FirebaseService { + final Database database; + + FirebaseService._({ + required this.database, + }); + + static Future create(Credentials credentials) async { + App app; + try { + app = FirebaseAdmin.instance.initializeApp( + AppOptions( + credential: + ServiceAccountCredential.fromJson(credentials.serviceAccount), + databaseUrl: credentials.databaseUrl, + ), + ); + } catch (e) { + // This will throw if Firebase has already been initialized + print(e); + app = FirebaseAdmin.instance.app()!; + } + + return FirebaseService._(database: app.database()); + } +} diff --git a/pub_stats_collector/pubspec.lock b/pub_stats_collector/pubspec.lock new file mode 100644 index 0000000..afb0f10 --- /dev/null +++ b/pub_stats_collector/pubspec.lock @@ -0,0 +1,878 @@ +# Generated by pub +# See https://dart.dev/tools/pub/glossary#lockfile +packages: + _discoveryapis_commons: + dependency: transitive + description: + name: _discoveryapis_commons + sha256: f8bb1fdbd77f3d5c1d62b5b0eca75fbf1e41bf4f6c62628f880582e2182ae45d + url: "https://pub.dev" + source: hosted + version: "1.0.6" + _fe_analyzer_shared: + dependency: transitive + description: + name: _fe_analyzer_shared + sha256: "36a321c3d2cbe01cbcb3540a87b8843846e0206df3e691fa7b23e19e78de6d49" + url: "https://pub.dev" + source: hosted + version: "65.0.0" + analyzer: + dependency: transitive + description: + name: analyzer + sha256: dfe03b90ec022450e22513b5e5ca1f01c0c01de9c3fba2f7fd233cb57a6b9a07 + url: "https://pub.dev" + source: hosted + version: "6.3.0" + args: + dependency: transitive + description: + name: args + sha256: eef6c46b622e0494a36c5a12d10d77fb4e855501a91c1b9ef9339326e58f0596 + url: "https://pub.dev" + source: hosted + version: "2.4.2" + asn1lib: + dependency: transitive + description: + name: asn1lib + sha256: "21afe4333076c02877d14f4a89df111e658a6d466cbfc802eb705eb91bd5adfd" + url: "https://pub.dev" + source: hosted + version: "1.5.0" + async: + dependency: transitive + description: + name: async + sha256: "947bfcf187f74dbc5e146c9eb9c0f10c9f8b30743e341481c1e2ed3ecc18c20c" + url: "https://pub.dev" + source: hosted + version: "2.11.0" + boolean_selector: + dependency: transitive + description: + name: boolean_selector + sha256: "6cfb5af12253eaf2b368f07bacc5a80d1301a071c73360d746b7f2e32d762c66" + url: "https://pub.dev" + source: hosted + version: "2.1.1" + build: + dependency: transitive + description: + name: build + sha256: "80184af8b6cb3e5c1c4ec6d8544d27711700bc3e6d2efad04238c7b5290889f0" + url: "https://pub.dev" + source: hosted + version: "2.4.1" + build_config: + dependency: transitive + description: + name: build_config + sha256: bf80fcfb46a29945b423bd9aad884590fb1dc69b330a4d4700cac476af1708d1 + url: "https://pub.dev" + source: hosted + version: "1.1.1" + build_daemon: + dependency: transitive + description: + name: build_daemon + sha256: "0343061a33da9c5810b2d6cee51945127d8f4c060b7fbdd9d54917f0a3feaaa1" + url: "https://pub.dev" + source: hosted + version: "4.0.1" + build_resolvers: + dependency: transitive + description: + name: build_resolvers + sha256: "64e12b0521812d1684b1917bc80945625391cb9bdd4312536b1d69dcb6133ed8" + url: "https://pub.dev" + source: hosted + version: "2.4.1" + build_runner: + dependency: "direct dev" + description: + name: build_runner + sha256: "10c6bcdbf9d049a0b666702cf1cee4ddfdc38f02a19d35ae392863b47519848b" + url: "https://pub.dev" + source: hosted + version: "2.4.6" + build_runner_core: + dependency: transitive + description: + name: build_runner_core + sha256: c9e32d21dd6626b5c163d48b037ce906bbe428bc23ab77bcd77bb21e593b6185 + url: "https://pub.dev" + source: hosted + version: "7.2.11" + built_collection: + dependency: transitive + description: + name: built_collection + sha256: "376e3dd27b51ea877c28d525560790aee2e6fbb5f20e2f85d5081027d94e2100" + url: "https://pub.dev" + source: hosted + version: "5.1.1" + built_value: + dependency: transitive + description: + name: built_value + sha256: "723b4021e903217dfc445ec4cf5b42e27975aece1fc4ebbc1ca6329c2d9fb54e" + url: "https://pub.dev" + source: hosted + version: "8.7.0" + checked_yaml: + dependency: transitive + description: + name: checked_yaml + sha256: feb6bed21949061731a7a75fc5d2aa727cf160b91af9a3e464c5e3a32e28b5ff + url: "https://pub.dev" + source: hosted + version: "2.0.3" + clock: + dependency: transitive + description: + name: clock + sha256: cb6d7f03e1de671e34607e909a7213e31d7752be4fb66a86d29fe1eb14bfb5cf + url: "https://pub.dev" + source: hosted + version: "1.1.1" + code_builder: + dependency: transitive + description: + name: code_builder + sha256: "1be9be30396d7e4c0db42c35ea6ccd7cc6a1e19916b5dc64d6ac216b5544d677" + url: "https://pub.dev" + source: hosted + version: "4.7.0" + collection: + dependency: transitive + description: + name: collection + sha256: ee67cb0715911d28db6bf4af1026078bd6f0128b07a5f66fb2ed94ec6783c09a + url: "https://pub.dev" + source: hosted + version: "1.18.0" + convert: + dependency: transitive + description: + name: convert + sha256: "0f08b14755d163f6e2134cb58222dd25ea2a2ee8a195e53983d57c075324d592" + url: "https://pub.dev" + source: hosted + version: "3.1.1" + coverage: + dependency: transitive + description: + name: coverage + sha256: ac86d3abab0f165e4b8f561280ff4e066bceaac83c424dd19f1ae2c2fcd12ca9 + url: "https://pub.dev" + source: hosted + version: "1.7.1" + crypto: + dependency: transitive + description: + name: crypto + sha256: ff625774173754681d66daaf4a448684fb04b78f902da9cb3d308c19cc5e8bab + url: "https://pub.dev" + source: hosted + version: "3.0.3" + crypto_keys: + dependency: transitive + description: + name: crypto_keys + sha256: acc19abf34623d990a0e8aec69463d74a824c31f137128f42e2810befc509ad0 + url: "https://pub.dev" + source: hosted + version: "0.3.0+1" + dart_style: + dependency: transitive + description: + name: dart_style + sha256: abd7625e16f51f554ea244d090292945ec4d4be7bfbaf2ec8cccea568919d334 + url: "https://pub.dev" + source: hosted + version: "2.3.3" + dio: + dependency: transitive + description: + name: dio + sha256: "417e2a6f9d83ab396ec38ff4ea5da6c254da71e4db765ad737a42af6930140b7" + url: "https://pub.dev" + source: hosted + version: "5.3.3" + dio_response_validator: + dependency: transitive + description: + name: dio_response_validator + sha256: "393f952894573b94303e1f798f77a04d69dce1e1c685abaf15ff01c5c61614c1" + url: "https://pub.dev" + source: hosted + version: "0.2.3" + discord_interactions: + dependency: "direct main" + description: + name: discord_interactions + sha256: a91641a919211a4a8f1ca03dac772216bc17574acf21289c4f0849da2f7dd534 + url: "https://pub.dev" + source: hosted + version: "0.0.20" + dotenv: + dependency: transitive + description: + name: dotenv + sha256: "379e64b6fc82d3df29461d349a1796ecd2c436c480d4653f3af6872eccbc90e1" + url: "https://pub.dev" + source: hosted + version: "4.2.0" + equatable: + dependency: transitive + description: + name: equatable + sha256: c2b87cb7756efdf69892005af546c56c0b5037f54d2a88269b4f347a505e3ca2 + url: "https://pub.dev" + source: hosted + version: "2.0.5" + expressions: + dependency: transitive + description: + name: expressions + sha256: "0a3a207dc4697bbb0e17c731c7f10f2df6f88d9dbb99ac83c4168a10b57e7a20" + url: "https://pub.dev" + source: hosted + version: "0.2.5+1" + file: + dependency: transitive + description: + name: file + sha256: "5fc22d7c25582e38ad9a8515372cd9a93834027aacf1801cf01164dac0ffa08c" + url: "https://pub.dev" + source: hosted + version: "7.0.0" + firebase_admin: + dependency: "direct main" + description: + path: "." + ref: "584ab8c108e3e2938127bd224fb89b90b2e8b6c5" + resolved-ref: "584ab8c108e3e2938127bd224fb89b90b2e8b6c5" + url: "https://github.com/Rexios80/firebase_admin" + source: git + version: "0.3.0-dev.4" + firebase_dart: + dependency: transitive + description: + name: firebase_dart + sha256: a65039f23dd699e52d05edba5404a055c25db97b1e8a0958516189a4c116ae93 + url: "https://pub.dev" + source: hosted + version: "1.1.1" + firebaseapis: + dependency: transitive + description: + name: firebaseapis + sha256: "7e9663f298a209dc7d4e8a10944d6c39d43e02e9cade08c915720768d02d712d" + url: "https://pub.dev" + source: hosted + version: "0.2.0" + fixnum: + dependency: transitive + description: + name: fixnum + sha256: "25517a4deb0c03aa0f32fd12db525856438902d9c16536311e76cdc57b31d7d1" + url: "https://pub.dev" + source: hosted + version: "1.1.0" + flutter_lints: + dependency: transitive + description: + name: flutter_lints + sha256: e2a421b7e59244faef694ba7b30562e489c2b489866e505074eb005cd7060db7 + url: "https://pub.dev" + source: hosted + version: "3.0.1" + flutter_tools_task_queue: + dependency: "direct main" + description: + name: flutter_tools_task_queue + sha256: bc1fdb4ff82f47ec21bcdd23992e6ef5e092890e5c15ef4707012a3e1a956a00 + url: "https://pub.dev" + source: hosted + version: "1.0.0" + frontend_server_client: + dependency: transitive + description: + name: frontend_server_client + sha256: "408e3ca148b31c20282ad6f37ebfa6f4bdc8fede5b74bc2f08d9d92b55db3612" + url: "https://pub.dev" + source: hosted + version: "3.2.0" + functions_framework: + dependency: "direct main" + description: + name: functions_framework + sha256: e40725fd249690d97319a2e96a0ef82f0510dffc6c99e22a0ebde4a26bddd7a9 + url: "https://pub.dev" + source: hosted + version: "0.4.3+1" + functions_framework_builder: + dependency: "direct dev" + description: + name: functions_framework_builder + sha256: dfd788517985a7e7f85deeb824128a067a2427592cb7f6a2ec49e17bdbfe4d96 + url: "https://pub.dev" + source: hosted + version: "0.4.10" + gcloud: + dependency: transitive + description: + name: gcloud + sha256: "94cc7901dba3bcc9cd1c0530caf2d87e733392ce8f93734ccb23d82ef9f2a3e2" + url: "https://pub.dev" + source: hosted + version: "0.8.11" + glob: + dependency: transitive + description: + name: glob + sha256: "0e7014b3b7d4dac1ca4d6114f82bf1782ee86745b9b42a92c9289c23d8a0ab63" + url: "https://pub.dev" + source: hosted + version: "2.1.2" + google_cloud: + dependency: transitive + description: + name: google_cloud + sha256: "2fa5d1adf3939ac6e7130ae84de40727efea9d0e08db2e3d7fea3b8b432d741c" + url: "https://pub.dev" + source: hosted + version: "0.2.0" + googleapis: + dependency: transitive + description: + name: googleapis + sha256: c2f311bcd1b3e4052234162edc6b626a9013a7e1385d6aad37e9e6a6c5f89908 + url: "https://pub.dev" + source: hosted + version: "11.4.0" + graphs: + dependency: transitive + description: + name: graphs + sha256: aedc5a15e78fc65a6e23bcd927f24c64dd995062bcd1ca6eda65a3cff92a4d19 + url: "https://pub.dev" + source: hosted + version: "2.3.1" + hive: + dependency: transitive + description: + name: hive + sha256: "8dcf6db979d7933da8217edcec84e9df1bdb4e4edc7fc77dbd5aa74356d6d941" + url: "https://pub.dev" + source: hosted + version: "2.2.3" + http: + dependency: transitive + description: + name: http + sha256: "759d1a329847dd0f39226c688d3e06a6b8679668e350e2891a6474f8b4bb8525" + url: "https://pub.dev" + source: hosted + version: "1.1.0" + http_multi_server: + dependency: transitive + description: + name: http_multi_server + sha256: "97486f20f9c2f7be8f514851703d0119c3596d14ea63227af6f7a481ef2b2f8b" + url: "https://pub.dev" + source: hosted + version: "3.2.1" + http_parser: + dependency: transitive + description: + name: http_parser + sha256: "2aa08ce0341cc9b354a498388e30986515406668dbcc4f7c950c3e715496693b" + url: "https://pub.dev" + source: hosted + version: "4.0.2" + intl: + dependency: transitive + description: + name: intl + sha256: "910f85bce16fb5c6f614e117efa303e85a1731bb0081edf3604a2ae6e9a3cc91" + url: "https://pub.dev" + source: hosted + version: "0.17.0" + io: + dependency: transitive + description: + name: io + sha256: "2ec25704aba361659e10e3e5f5d672068d332fc8ac516421d483a11e5cbd061e" + url: "https://pub.dev" + source: hosted + version: "1.0.4" + jose: + dependency: transitive + description: + name: jose + sha256: "7955ec5d131960104e81fbf151abacb9d835c16c9e793ed394b2809f28b2198d" + url: "https://pub.dev" + source: hosted + version: "0.3.4" + js: + dependency: transitive + description: + name: js + sha256: f2c445dce49627136094980615a031419f7f3eb393237e4ecd97ac15dea343f3 + url: "https://pub.dev" + source: hosted + version: "0.6.7" + json_annotation: + dependency: transitive + description: + name: json_annotation + sha256: b10a7b2ff83d83c777edba3c6a0f97045ddadd56c944e1a23a3fdf43a1bf4467 + url: "https://pub.dev" + source: hosted + version: "4.8.1" + lints: + dependency: transitive + description: + name: lints + sha256: cbf8d4b858bb0134ef3ef87841abdf8d63bfc255c266b7bf6b39daa1085c4290 + url: "https://pub.dev" + source: hosted + version: "3.0.0" + logging: + dependency: transitive + description: + name: logging + sha256: "623a88c9594aa774443aa3eb2d41807a48486b5613e67599fb4c41c0ad47c340" + url: "https://pub.dev" + source: hosted + version: "1.2.0" + matcher: + dependency: transitive + description: + name: matcher + sha256: "1803e76e6653768d64ed8ff2e1e67bea3ad4b923eb5c56a295c3e634bad5960e" + url: "https://pub.dev" + source: hosted + version: "0.12.16" + meta: + dependency: transitive + description: + name: meta + sha256: d584fa6707a52763a52446f02cc621b077888fb63b93bbcb1143a7be5a0c0c04 + url: "https://pub.dev" + source: hosted + version: "1.11.0" + mime: + dependency: transitive + description: + name: mime + sha256: e4ff8e8564c03f255408decd16e7899da1733852a9110a58fe6d1b817684a63e + url: "https://pub.dev" + source: hosted + version: "1.0.4" + node_preamble: + dependency: transitive + description: + name: node_preamble + sha256: "6e7eac89047ab8a8d26cf16127b5ed26de65209847630400f9aefd7cd5c730db" + url: "https://pub.dev" + source: hosted + version: "2.0.2" + oauth2: + dependency: transitive + description: + name: oauth2 + sha256: c4013ef62be37744efdc0861878fd9e9285f34db1f9e331cc34100d7674feb42 + url: "https://pub.dev" + source: hosted + version: "2.0.2" + openid_client: + dependency: transitive + description: + name: openid_client + sha256: "043878e907b7a1b460b54fb7b3b27b101cf70d4ac28b32a2db87ae67dbaed611" + url: "https://pub.dev" + source: hosted + version: "0.4.8" + package_config: + dependency: transitive + description: + name: package_config + sha256: "1c5b77ccc91e4823a5af61ee74e6b972db1ef98c2ff5a18d3161c982a55448bd" + url: "https://pub.dev" + source: hosted + version: "2.1.0" + path: + dependency: transitive + description: + name: path + sha256: "8829d8a55c13fc0e37127c29fedf290c102f4e40ae94ada574091fe0ff96c917" + url: "https://pub.dev" + source: hosted + version: "1.8.3" + petitparser: + dependency: transitive + description: + name: petitparser + sha256: eeb2d1428ee7f4170e2bd498827296a18d4e7fc462b71727d111c0ac7707cfa6 + url: "https://pub.dev" + source: hosted + version: "6.0.1" + pinenacl: + dependency: transitive + description: + name: pinenacl + sha256: "3a5503637587d635647c93ea9a8fecf48a420cc7deebe6f1fc85c2a5637ab327" + url: "https://pub.dev" + source: hosted + version: "0.5.1" + pointycastle: + dependency: transitive + description: + name: pointycastle + sha256: "7c1e5f0d23c9016c5bbd8b1473d0d3fb3fc851b876046039509e18e0c7485f2c" + url: "https://pub.dev" + source: hosted + version: "3.7.3" + pool: + dependency: transitive + description: + name: pool + sha256: "20fe868b6314b322ea036ba325e6fc0711a22948856475e2c2b6306e8ab39c2a" + url: "https://pub.dev" + source: hosted + version: "1.5.1" + pub_api_client: + dependency: "direct main" + description: + path: "." + ref: "47b53a0d1e8380f6152ef5b4458b0acf511d5517" + resolved-ref: "47b53a0d1e8380f6152ef5b4458b0acf511d5517" + url: "https://github.com/Rexios80/pub_api_client" + source: git + version: "2.6.0" + pub_semver: + dependency: transitive + description: + name: pub_semver + sha256: "40d3ab1bbd474c4c2328c91e3a7df8c6dd629b79ece4c4bd04bee496a224fb0c" + url: "https://pub.dev" + source: hosted + version: "2.1.4" + pub_stats_core: + dependency: "direct main" + description: + path: "../pub_stats_core" + relative: true + source: path + version: "1.0.0" + pubspec: + dependency: transitive + description: + name: pubspec + sha256: f534a50a2b4d48dc3bc0ec147c8bd7c304280fff23b153f3f11803c4d49d927e + url: "https://pub.dev" + source: hosted + version: "2.3.0" + pubspec_parse: + dependency: transitive + description: + name: pubspec_parse + sha256: c63b2876e58e194e4b0828fcb080ad0e06d051cb607a6be51a9e084f47cb9367 + url: "https://pub.dev" + source: hosted + version: "1.2.3" + quiver: + dependency: transitive + description: + name: quiver + sha256: b1c1ac5ce6688d77f65f3375a9abb9319b3cb32486bdc7a1e0fdf004d7ba4e47 + url: "https://pub.dev" + source: hosted + version: "3.2.1" + recase: + dependency: "direct main" + description: + name: recase + sha256: e4eb4ec2dcdee52dcf99cb4ceabaffc631d7424ee55e56f280bc039737f89213 + url: "https://pub.dev" + source: hosted + version: "4.1.0" + retry: + dependency: transitive + description: + name: retry + sha256: "822e118d5b3aafed083109c72d5f484c6dc66707885e07c0fbcb8b986bba7efc" + url: "https://pub.dev" + source: hosted + version: "3.1.2" + rexios_lints: + dependency: "direct dev" + description: + name: rexios_lints + sha256: "0e099c3ab11dca537ceef7668c2f9c50650c0551cf59858db4559ddaedcfbfd9" + url: "https://pub.dev" + source: hosted + version: "6.0.1" + rxdart: + dependency: transitive + description: + name: rxdart + sha256: "0c7c0cedd93788d996e33041ffecda924cc54389199cde4e6a34b440f50044cb" + url: "https://pub.dev" + source: hosted + version: "0.27.7" + shelf: + dependency: "direct main" + description: + name: shelf + sha256: ad29c505aee705f41a4d8963641f91ac4cee3c8fad5947e033390a7bd8180fa4 + url: "https://pub.dev" + source: hosted + version: "1.4.1" + shelf_packages_handler: + dependency: transitive + description: + name: shelf_packages_handler + sha256: "89f967eca29607c933ba9571d838be31d67f53f6e4ee15147d5dc2934fee1b1e" + url: "https://pub.dev" + source: hosted + version: "3.0.2" + shelf_static: + dependency: transitive + description: + name: shelf_static + sha256: a41d3f53c4adf0f57480578c1d61d90342cd617de7fc8077b1304643c2d85c1e + url: "https://pub.dev" + source: hosted + version: "1.1.2" + shelf_web_socket: + dependency: transitive + description: + name: shelf_web_socket + sha256: "9ca081be41c60190ebcb4766b2486a7d50261db7bd0f5d9615f2d653637a84c1" + url: "https://pub.dev" + source: hosted + version: "1.0.4" + snapshot: + dependency: transitive + description: + name: snapshot + sha256: "04a7910492987d00ca8c6767297ed773e6e206d0fc1ce7365a3cddf6847e92b5" + url: "https://pub.dev" + source: hosted + version: "0.2.5" + sortedmap: + dependency: transitive + description: + name: sortedmap + sha256: "8c9e19b93c7e1b4d795776a02719df229606cb57f46f857a7f6eb80c97fc0600" + url: "https://pub.dev" + source: hosted + version: "0.5.3" + source_gen: + dependency: transitive + description: + name: source_gen + sha256: fc0da689e5302edb6177fdd964efcb7f58912f43c28c2047a808f5bfff643d16 + url: "https://pub.dev" + source: hosted + version: "1.4.0" + source_helper: + dependency: transitive + description: + name: source_helper + sha256: "6adebc0006c37dd63fe05bca0a929b99f06402fc95aa35bf36d67f5c06de01fd" + url: "https://pub.dev" + source: hosted + version: "1.3.4" + source_map_stack_trace: + dependency: transitive + description: + name: source_map_stack_trace + sha256: "84cf769ad83aa6bb61e0aa5a18e53aea683395f196a6f39c4c881fb90ed4f7ae" + url: "https://pub.dev" + source: hosted + version: "2.1.1" + source_maps: + dependency: transitive + description: + name: source_maps + sha256: "708b3f6b97248e5781f493b765c3337db11c5d2c81c3094f10904bfa8004c703" + url: "https://pub.dev" + source: hosted + version: "0.10.12" + source_span: + dependency: transitive + description: + name: source_span + sha256: "53e943d4206a5e30df338fd4c6e7a077e02254531b138a15aec3bd143c1a8b3c" + url: "https://pub.dev" + source: hosted + version: "1.10.0" + sprintf: + dependency: transitive + description: + name: sprintf + sha256: "1fc9ffe69d4df602376b52949af107d8f5703b77cda567c4d7d86a0693120f23" + url: "https://pub.dev" + source: hosted + version: "7.0.0" + stack_trace: + dependency: transitive + description: + name: stack_trace + sha256: "73713990125a6d93122541237550ee3352a2d84baad52d375a4cad2eb9b7ce0b" + url: "https://pub.dev" + source: hosted + version: "1.11.1" + stream_channel: + dependency: transitive + description: + name: stream_channel + sha256: ba2aa5d8cc609d96bbb2899c28934f9e1af5cddbd60a827822ea467161eb54e7 + url: "https://pub.dev" + source: hosted + version: "2.1.2" + stream_transform: + dependency: transitive + description: + name: stream_transform + sha256: "14a00e794c7c11aa145a170587321aedce29769c08d7f58b1d141da75e3b1c6f" + url: "https://pub.dev" + source: hosted + version: "2.1.0" + string_scanner: + dependency: transitive + description: + name: string_scanner + sha256: "556692adab6cfa87322a115640c11f13cb77b3f076ddcc5d6ae3c20242bedcde" + url: "https://pub.dev" + source: hosted + version: "1.2.0" + synchronized: + dependency: transitive + description: + name: synchronized + sha256: "5fcbd27688af6082f5abd611af56ee575342c30e87541d0245f7ff99faa02c60" + url: "https://pub.dev" + source: hosted + version: "3.1.0" + term_glyph: + dependency: transitive + description: + name: term_glyph + sha256: a29248a84fbb7c79282b40b8c72a1209db169a2e0542bce341da992fe1bc7e84 + url: "https://pub.dev" + source: hosted + version: "1.2.1" + test: + dependency: "direct dev" + description: + name: test + sha256: a1f7595805820fcc05e5c52e3a231aedd0b72972cb333e8c738a8b1239448b6f + url: "https://pub.dev" + source: hosted + version: "1.24.9" + test_api: + dependency: transitive + description: + name: test_api + sha256: "5c2f730018264d276c20e4f1503fd1308dfbbae39ec8ee63c5236311ac06954b" + url: "https://pub.dev" + source: hosted + version: "0.6.1" + test_core: + dependency: transitive + description: + name: test_core + sha256: a757b14fc47507060a162cc2530d9a4a2f92f5100a952c7443b5cad5ef5b106a + url: "https://pub.dev" + source: hosted + version: "0.5.9" + timing: + dependency: transitive + description: + name: timing + sha256: "70a3b636575d4163c477e6de42f247a23b315ae20e86442bebe32d3cabf61c32" + url: "https://pub.dev" + source: hosted + version: "1.0.1" + typed_data: + dependency: transitive + description: + name: typed_data + sha256: facc8d6582f16042dd49f2463ff1bd6e2c9ef9f3d5da3d9b087e244a7b564b3c + url: "https://pub.dev" + source: hosted + version: "1.3.2" + uri: + dependency: transitive + description: + name: uri + sha256: "889eea21e953187c6099802b7b4cf5219ba8f3518f604a1033064d45b1b8268a" + url: "https://pub.dev" + source: hosted + version: "1.0.0" + uuid: + dependency: transitive + description: + name: uuid + sha256: df5a4d8f22ee4ccd77f8839ac7cb274ebc11ef9adcce8b92be14b797fe889921 + url: "https://pub.dev" + source: hosted + version: "4.2.1" + vm_service: + dependency: transitive + description: + name: vm_service + sha256: b3d56ff4341b8f182b96aceb2fa20e3dcb336b9f867bc0eafc0de10f1048e957 + url: "https://pub.dev" + source: hosted + version: "13.0.0" + watcher: + dependency: transitive + description: + name: watcher + sha256: "3d2ad6751b3c16cf07c7fca317a1413b3f26530319181b37e3b9039b84fc01d8" + url: "https://pub.dev" + source: hosted + version: "1.1.0" + web_socket_channel: + dependency: transitive + description: + name: web_socket_channel + sha256: d88238e5eac9a42bb43ca4e721edba3c08c6354d4a53063afaa568516217621b + url: "https://pub.dev" + source: hosted + version: "2.4.0" + webkit_inspection_protocol: + dependency: transitive + description: + name: webkit_inspection_protocol + sha256: "87d3f2333bb240704cd3f1c6b5b7acd8a10e7f0bc28c28dcf14e782014f4a572" + url: "https://pub.dev" + source: hosted + version: "1.2.1" + x509: + dependency: transitive + description: + name: x509 + sha256: f9ac84e137edc719767fb218b8d72b1ecc5b5e29ad607b79b253d1a517ca5ba8 + url: "https://pub.dev" + source: hosted + version: "0.2.4" + yaml: + dependency: transitive + description: + name: yaml + sha256: "75769501ea3489fca56601ff33454fe45507ea3bfb014161abc3b43ae25989d5" + url: "https://pub.dev" + source: hosted + version: "3.1.2" +sdks: + dart: ">=3.0.0 <4.0.0" diff --git a/pub_stats_collector/pubspec.yaml b/pub_stats_collector/pubspec.yaml new file mode 100644 index 0000000..2e25a58 --- /dev/null +++ b/pub_stats_collector/pubspec.yaml @@ -0,0 +1,32 @@ +name: pub_stats_collector +version: 1.0.0 +publish_to: none + +environment: + sdk: ^3.0.0 + +dependencies: + functions_framework: ^0.4.1 + pub_stats_core: + path: ../pub_stats_core + firebase_admin: + git: + url: https://github.com/Rexios80/firebase_admin + ref: 584ab8c108e3e2938127bd224fb89b90b2e8b6c5 + pub_api_client: ^2.4.0 + shelf: ^1.4.0 + discord_interactions: ^0.0.15 + flutter_tools_task_queue: ^1.0.0 + recase: ^4.1.0 + +dev_dependencies: + rexios_lints: ^6.0.1 + test: ^1.20.1 + build_runner: ^2.1.7 + functions_framework_builder: ^0.4.5 + +dependency_overrides: + pub_api_client: + git: + url: https://github.com/Rexios80/pub_api_client + ref: 47b53a0d1e8380f6152ef5b4458b0acf511d5517 diff --git a/pub_stats_collector/test/fetch_test.dart b/pub_stats_collector/test/fetch_test.dart new file mode 100644 index 0000000..64521a4 --- /dev/null +++ b/pub_stats_collector/test/fetch_test.dart @@ -0,0 +1,9 @@ +import 'package:pub_stats_collector/functions.dart' as functions; +import 'package:shelf/shelf.dart' as shelf; + +void main() async { + await functions.function( + shelf.Request('GET', Uri.parse('https://pubstats.dev/test')), + debug: true, + ); +} diff --git a/pub_stats_collector/tool/prune.dart b/pub_stats_collector/tool/prune.dart new file mode 100644 index 0000000..cddf70f --- /dev/null +++ b/pub_stats_collector/tool/prune.dart @@ -0,0 +1,67 @@ +import 'dart:convert'; +import 'dart:io'; + +import 'package:pub_stats_core/pub_stats_core.dart'; + +/// Prune duplicate score entries from the database +void main() { + final string = File( + '/Users/rexios/Downloads/pub-stats-collector-default-rtdb-export.json', + ).readAsStringSync(); + final json = jsonDecode(string) as Map; + final packages = json['stats'] as Map; + + var beforeTotal = 0; + var afterTotal = 0; + for (final package in packages.keys) { + final scoresJson = packages[package] as Map; + final scores = scoresJson.map( + (k, v) => MapEntry(k, MiniPackageScore.fromJson(v)), + ); + + final before = scores.length; + beforeTotal += before; + prune(scores); + final after = scores.length; + afterTotal += after; + final diff = before - after; + print('Pruned $diff entries from $package'); + } + + final newString = jsonEncode(json); + + final beforeMegabytes = stringToMegabytes(string); + final afterMegabytes = stringToMegabytes(newString); + + final diff = beforeTotal - afterTotal; + final diffMegabytes = beforeMegabytes - afterMegabytes; + + print('Before: $beforeTotal ($beforeMegabytes MB)'); + print('After: $afterTotal ($afterMegabytes MB)'); + print('Pruned: $diff ($diffMegabytes MB)'); + + File('pruned.json').writeAsStringSync(newString); +} + +void prune(Map scores) { + final collisions = {}; + var current = scores.entries.first; + for (final next in scores.entries.skip(1)) { + final currentValue = current.value; + final nextValue = next.value; + if (currentValue.likeCount != nextValue.likeCount || + currentValue.popularityScore != nextValue.popularityScore) { + current = next; + continue; + } + collisions.add(next.key); + } + + for (final collision in collisions) { + scores.remove(collision); + } +} + +int stringToMegabytes(String string) { + return string.codeUnits.length / 1024 ~/ 1024; +}