Provides an alternative implementation on Redis Streams

Implements backlogs using streams rather than sorted sets. Streams do their own length limitation. See https://redis.io/topics/streams-intro.
discourse · Nov 8, 2018 · baf8e39 · baf8e39
1 parent 8eba0cb
commit baf8e39
Show file tree

Hide file tree

Showing 6 changed files with 473 additions and 33 deletions.
diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml
@@ -1,6 +1,6 @@
 # This configuration was generated by
 # `rubocop --auto-gen-config`
-# on 2018-11-07 23:41:56 +0000 using RuboCop version 0.60.0.
+# on 2018-11-08 00:58:02 +0000 using RuboCop version 0.60.0.
 # The point is for the user to remove these configuration records
 # one by one as the offenses are removed from the code base.
 # Note that changes in the inspected code, or installation of new
@@ -54,12 +54,13 @@ Lint/LiteralAsCondition:
   Exclude:
     - 'examples/chat/chat.rb'
 
-# Offense count: 3
+# Offense count: 4
 Lint/NonLocalExitFromIterator:
   Exclude:
     - 'lib/message_bus.rb'
     - 'lib/message_bus/backends/postgres.rb'
     - 'lib/message_bus/backends/redis.rb'
+    - 'lib/message_bus/backends/redis_streams.rb'
 
 # Offense count: 1
 # Cop supports --auto-correct.
@@ -78,7 +79,7 @@ Lint/StringConversionInInterpolation:
   Exclude:
     - 'spec/lib/message_bus/multi_process_spec.rb'
 
-# Offense count: 30
+# Offense count: 36
 Metrics/AbcSize:
   Max: 112
 
@@ -88,21 +89,21 @@ Metrics/AbcSize:
 Metrics/BlockLength:
   Max: 289
 
-# Offense count: 3
+# Offense count: 4
 # Configuration parameters: CountBlocks.
 Metrics/BlockNesting:
   Max: 4
 
-# Offense count: 8
+# Offense count: 10
 # Configuration parameters: CountComments.
 Metrics/ClassLength:
-  Max: 307
+  Max: 312
 
-# Offense count: 18
+# Offense count: 21
 Metrics/CyclomaticComplexity:
   Max: 30
 
-# Offense count: 39
+# Offense count: 46
 # Configuration parameters: CountComments, ExcludedMethods.
 Metrics/MethodLength:
   Max: 105
@@ -112,7 +113,7 @@ Metrics/MethodLength:
 Metrics/ModuleLength:
   Max: 415
 
-# Offense count: 18
+# Offense count: 21
 Metrics/PerceivedComplexity:
   Max: 34
 
@@ -124,7 +125,7 @@ Naming/MemoizedInstanceVariableName:
     - 'lib/message_bus.rb'
     - 'spec/assets/support/jasmine_helper.rb'
 
-# Offense count: 2
+# Offense count: 3
 # Configuration parameters: NamePrefix, NamePrefixBlacklist, NameWhitelist, MethodDefinitionMacros.
 # NamePrefix: is_, has_, have_
 # NamePrefixBlacklist: is_, has_, have_
@@ -135,6 +136,7 @@ Naming/PredicateName:
     - 'spec/**/*'
     - 'lib/message_bus.rb'
     - 'lib/message_bus/backends/redis.rb'
+    - 'lib/message_bus/backends/redis_streams.rb'
 
 # Offense count: 6
 # Configuration parameters: MinNameLength, AllowNamesEndingInNumbers, AllowedNames, ForbiddenNames.
@@ -158,11 +160,12 @@ Performance/RedundantMerge:
   Exclude:
     - 'spec/spec_helper.rb'
 
-# Offense count: 3
+# Offense count: 5
 # Cop supports --auto-correct.
 Performance/RegexpMatch:
   Exclude:
     - 'lib/message_bus/backends/redis.rb'
+    - 'lib/message_bus/backends/redis_streams.rb'
     - 'lib/message_bus/rack/diagnostics.rb'
 
 # Offense count: 1
@@ -211,7 +214,7 @@ Style/CaseEquality:
     - 'lib/message_bus/distributed_cache.rb'
     - 'spec/lib/message_bus/rack/middleware_spec.rb'
 
-# Offense count: 24
+# Offense count: 26
 # Cop supports --auto-correct.
 # Configuration parameters: AutoCorrect, EnforcedStyle.
 # SupportedStyles: nested, compact
@@ -221,6 +224,7 @@ Style/ClassAndModuleChildren:
     - 'lib/message_bus/backends/memory.rb'
     - 'lib/message_bus/backends/postgres.rb'
     - 'lib/message_bus/backends/redis.rb'
+    - 'lib/message_bus/backends/redis_streams.rb'
     - 'lib/message_bus/client.rb'
     - 'lib/message_bus/connection_manager.rb'
     - 'lib/message_bus/diagnostics.rb'
@@ -236,21 +240,22 @@ Style/ClassVars:
   Exclude:
     - 'lib/message_bus/rack/diagnostics.rb'
 
-# Offense count: 3
+# Offense count: 4
 # Cop supports --auto-correct.
 # Configuration parameters: Keywords.
 # Keywords: TODO, FIXME, OPTIMIZE, HACK, REVIEW
 Style/CommentAnnotation:
   Exclude:
     - 'lib/message_bus/backends/postgres.rb'
     - 'lib/message_bus/backends/redis.rb'
+    - 'lib/message_bus/backends/redis_streams.rb'
     - 'lib/message_bus/rack/middleware.rb'
 
-# Offense count: 33
+# Offense count: 37
 Style/Documentation:
   Enabled: false
 
-# Offense count: 6
+# Offense count: 7
 # Cop supports --auto-correct.
 # Configuration parameters: EnforcedStyle.
 # SupportedStyles: empty, nil, both
@@ -260,6 +265,7 @@ Style/EmptyElse:
     - 'lib/message_bus/backends/memory.rb'
     - 'lib/message_bus/backends/postgres.rb'
     - 'lib/message_bus/backends/redis.rb'
+    - 'lib/message_bus/backends/redis_streams.rb'
     - 'spec/lib/message_bus/backend_spec.rb'
 
 # Offense count: 1
@@ -305,14 +311,15 @@ Style/GlobalVars:
     - 'examples/chat/chat.rb'
     - 'lib/message_bus/diagnostics.rb'
 
-# Offense count: 12
+# Offense count: 15
 # Configuration parameters: MinBodyLength.
 Style/GuardClause:
   Exclude:
     - 'lib/message_bus.rb'
     - 'lib/message_bus/backends/memory.rb'
     - 'lib/message_bus/backends/postgres.rb'
     - 'lib/message_bus/backends/redis.rb'
+    - 'lib/message_bus/backends/redis_streams.rb'
     - 'lib/message_bus/client.rb'
     - 'lib/message_bus/rack/middleware.rb'
 
@@ -321,7 +328,7 @@ Style/IfInsideElse:
   Exclude:
     - 'lib/message_bus.rb'
 
-# Offense count: 22
+# Offense count: 30
 # Cop supports --auto-correct.
 Style/IfUnlessModifier:
   Exclude:
@@ -330,6 +337,7 @@ Style/IfUnlessModifier:
     - 'lib/message_bus/backends/memory.rb'
     - 'lib/message_bus/backends/postgres.rb'
     - 'lib/message_bus/backends/redis.rb'
+    - 'lib/message_bus/backends/redis_streams.rb'
     - 'lib/message_bus/client.rb'
     - 'lib/message_bus/rack/diagnostics.rb'
     - 'lib/message_bus/rack/middleware.rb'
@@ -362,11 +370,12 @@ Style/MutableConstant:
     - 'lib/message_bus/backends/postgres.rb'
     - 'spec/spec_helper.rb'
 
-# Offense count: 2
+# Offense count: 3
 # Cop supports --auto-correct.
 Style/NegatedWhile:
   Exclude:
     - 'lib/message_bus/backends/redis.rb'
+    - 'lib/message_bus/backends/redis_streams.rb'
     - 'lib/message_bus/timer_thread.rb'
 
 # Offense count: 2
@@ -378,22 +387,23 @@ Style/Next:
     - 'lib/message_bus/connection_manager.rb'
     - 'lib/message_bus/timer_thread.rb'
 
-# Offense count: 4
+# Offense count: 6
 # Cop supports --auto-correct.
 # Configuration parameters: EnforcedStyle.
 # SupportedStyles: predicate, comparison
 Style/NilComparison:
   Exclude:
     - 'lib/message_bus/backends/redis.rb'
+    - 'lib/message_bus/backends/redis_streams.rb'
     - 'spec/lib/message_bus/distributed_cache_spec.rb'
 
-# Offense count: 5
+# Offense count: 7
 # Cop supports --auto-correct.
 # Configuration parameters: Strict.
 Style/NumericLiterals:
   MinDigits: 15
 
-# Offense count: 15
+# Offense count: 17
 # Cop supports --auto-correct.
 # Configuration parameters: AutoCorrect, EnforcedStyle, IgnoredMethods.
 # SupportedStyles: predicate, comparison
@@ -404,6 +414,7 @@ Style/NumericPredicate:
     - 'lib/message_bus/backends/memory.rb'
     - 'lib/message_bus/backends/postgres.rb'
     - 'lib/message_bus/backends/redis.rb'
+    - 'lib/message_bus/backends/redis_streams.rb'
     - 'lib/message_bus/client.rb'
     - 'lib/message_bus/rack/middleware.rb'
     - 'lib/message_bus/timer_thread.rb'
@@ -423,20 +434,22 @@ Style/PercentLiteralDelimiters:
     - 'Guardfile'
     - 'message_bus.gemspec'
 
-# Offense count: 2
+# Offense count: 3
 # Cop supports --auto-correct.
 # Configuration parameters: EnforcedStyle.
 # SupportedStyles: compact, exploded
 Style/RaiseArgs:
   Exclude:
     - 'lib/message_bus.rb'
     - 'lib/message_bus/backends/redis.rb'
+    - 'lib/message_bus/backends/redis_streams.rb'
 
-# Offense count: 3
+# Offense count: 4
 # Cop supports --auto-correct.
 Style/RedundantBegin:
   Exclude:
     - 'lib/message_bus/backends/redis.rb'
+    - 'lib/message_bus/backends/redis_streams.rb'
     - 'lib/message_bus/diagnostics.rb'
 
 # Offense count: 2
@@ -489,7 +502,7 @@ Style/RescueModifier:
     - 'lib/message_bus/client.rb'
     - 'lib/message_bus/connection_manager.rb'
 
-# Offense count: 17
+# Offense count: 19
 # Cop supports --auto-correct.
 # Configuration parameters: EnforcedStyle.
 # SupportedStyles: implicit, explicit
@@ -500,6 +513,7 @@ Style/RescueStandardError:
     - 'lib/message_bus/backends/memory.rb'
     - 'lib/message_bus/backends/postgres.rb'
     - 'lib/message_bus/backends/redis.rb'
+    - 'lib/message_bus/backends/redis_streams.rb'
     - 'lib/message_bus/client.rb'
     - 'lib/message_bus/connection_manager.rb'
     - 'lib/message_bus/diagnostics.rb'
@@ -540,7 +554,7 @@ Style/SpecialGlobalVars:
     - 'message_bus.gemspec'
     - 'spec/spec_helper.rb'
 
-# Offense count: 647
+# Offense count: 658
 # Cop supports --auto-correct.
 # Configuration parameters: EnforcedStyle, ConsistentQuotesInMultiline.
 # SupportedStyles: single_quotes, double_quotes
@@ -639,16 +653,17 @@ Style/YodaCondition:
   Exclude:
     - 'spec/lib/message_bus/timer_thread_spec.rb'
 
-# Offense count: 6
+# Offense count: 7
 # Cop supports --auto-correct.
 Style/ZeroLengthPredicate:
   Exclude:
     - 'lib/message_bus/backends/redis.rb'
+    - 'lib/message_bus/backends/redis_streams.rb'
     - 'lib/message_bus/client.rb'
     - 'lib/message_bus/rack/middleware.rb'
     - 'spec/lib/message_bus_spec.rb'
 
-# Offense count: 104
+# Offense count: 115
 # Configuration parameters: AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, IgnoredPatterns.
 # URISchemes: http, https
 Metrics/LineLength:

diff --git a/README.md b/README.md
@@ -323,6 +323,8 @@ MessageBus.reliable_pub_sub.max_backlog_age = 100
 
 ```
 
+The primary Redis-based implementation uses Redis PubSub and sorted sets. An alternative implementation based on [Redis Streams](https://redis.io/topics/streams-intro) (available in Redis 5.0) is available by setting `backend: :redis_streams`.
+
 ### PostgreSQL
 
 message_bus also supports PostgreSQL as the backend:

diff --git a/Rakefile b/Rakefile
@@ -37,14 +37,18 @@ run_spec = proc do |backend|
   end
 end
 
-task spec: [:spec_memory, :spec_redis, :spec_postgres, :spec_client_js, :rubocop]
+task spec: [:spec_memory, :spec_redis, :spec_redis_streams, :spec_postgres, :spec_client_js, :rubocop]
 
 task spec_client_js: 'jasmine:ci'
 
 task :spec_redis do
   run_spec.call('redis')
 end
 
+task :spec_redis_streams do
+  run_spec.call('redis_streams')
+end
+
 task :spec_memory do
   run_spec.call('memory')
 end
-Original file line number
+Diff line change
@@ Expand Up / @@ -323,6 +323,8 @@ MessageBus.reliable_pub_sub.max_backlog_age = 100 @@
     ```
+    The primary Redis-based implementation uses Redis PubSub and sorted sets. An alternative implementation based on [Redis Streams](https://redis.io/topics/streams-intro) (available in Redis 5.0) is available by setting `backend: :redis_streams`.
     ### PostgreSQL
     message_bus also supports PostgreSQL as the backend:
@@ Expand Down @@